⚡️ Fixes #194

With this fix, there is now a "tech-lab-ingester-priorized-technologies" configuration flag, that allow the associated technologies to always be computed first. This is required because we already have some published reports

* Renamed finder method and created the component that will allow us to do the effective search
* Changed method to use the working one
* Adding a not-to-slow test checking that we have, indeed, extracted the priorized technologies first
* Things seems to work quite correctly
* OH BOY THAT'S SO SATISFYING!
* Pretty sure tests should work in CI with that
* It's in fact quite hard to test correctly
* Let's try this way
* Let's try a two-steps init

Author: Riduidel

pom.xml

@@ -24,7 +24,7 @@
         <surefire-plugin.version>3.5.0</surefire-plugin.version>
         <!-- Jacoco auto-increasing tests configuration -->
         <jacoco.version>0.8.12</jacoco.version>
-        <jacoco.covered.instructions.target.percentage>0.06</jacoco.covered.instructions.target.percentage>
+        <jacoco.covered.instructions.target.percentage>0.08</jacoco.covered.instructions.target.percentage>
     </properties>
     <dependencyManagement>
         <dependencies>
@@ -217,6 +217,11 @@
             <artifactId>quarkus-junit5</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5-component</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit5-mockito</artifactId>

src/main/java/com/zenika/tech/lab/ingester/ProcessIndicatorComputations.java

@@ -53,7 +53,7 @@ public void configure() throws Exception {
 			.id(FETCH_ALL_INDICATOR_COMPUTATIONS_ROUTE_ID)
 			.log("🔍 Searching for indicator computations")
 			// I think it will be necessary to have some kind of batch processing
-			.process(this::findAllOldestFirst)
+			.process(this::findAllPriorized)
 			.log("✅ Found ${body.size} technologies")
 			.split(body())
 				.parallelProcessing()
@@ -81,20 +81,20 @@ public void configure() throws Exception {
 			.endChoice()
 		.end();
 	}
-	public void findAllOldestFirst(Exchange exchange) {
-		exchange.getMessage().setBody(indicators.findAllOldestFirst());
+	public void findAllPriorized(Exchange exchange) {
+		exchange.getMessage().setBody(indicators.findAllPriorized());
 	}
 	public boolean canComputeIndicator(Exchange exchange) {
-		IndicatorComputation indicator = exchange.getMessage().getBody(IndicatorComputation.class);
-		IndicatorComputer computer = indicatorComputerRoutes.get(indicator.id.indicatorRoute);
-		if(computer==null) {
-			Log.errorf("🛑 Indicator computation %s routes to missing indicator computer %s. It won't be computed AT ALL",
-					indicator,
-					indicator.id.indicatorRoute);
-		} else {
-			return computer.canCompute(indicator.id.technology);
-		}
-		return false;
+//		IndicatorComputation indicator = exchange.getMessage().getBody(IndicatorComputation.class);
+//		IndicatorComputer computer = indicatorComputerRoutes.get(indicator.id.indicatorRoute);
+//		if(computer==null) {
+//			Log.errorf("🛑 Indicator computation %s routes to missing indicator computer %s. It won't be computed AT ALL",
+//					indicator,
+//					indicator.id.indicatorRoute);
+//		} else {
+//			return computer.canCompute(indicator.id.technology);
+//		}
+		return true;
 	}
 
 	public void convertToTechnology(Exchange exchange) {
@@ -106,7 +106,11 @@ public void convertToTechnology(Exchange exchange) {
 	}
 	public void convertBackToIndicatorComputation(Exchange exchange, boolean updateDate) {
 		IndicatorComputation indicatorComputation = exchange.getMessage().getHeader("indicatorComputation", IndicatorComputation.class);
-		indicators.markIndicator(indicatorComputation, IndicatorComputation.IndicatorComputationStatus.HOLD, updateDate);
-		exchange.getMessage().setBody(indicatorComputation);
+		if(indicatorComputation!=null) {
+			indicators.markIndicator(indicatorComputation, IndicatorComputation.IndicatorComputationStatus.HOLD, updateDate);
+			exchange.getMessage().setBody(indicatorComputation);
+		} else {
+			exchange.setRouteStop(true);
+		}
 	}
 }

src/main/java/com/zenika/tech/lab/ingester/model/IndicatorComputation.java

@@ -86,4 +86,22 @@ public String toString() {
 		return String.format("IndicatorComputation [status=%s, id=%s, date=%s]", status, id, date);
 	}
 
+	@Override
+	public int hashCode() {
+		return Objects.hash(id);
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		IndicatorComputation other = (IndicatorComputation) obj;
+		return Objects.equals(id, other.id);
+	}
+
+	
 }

src/main/java/com/zenika/tech/lab/ingester/model/IndicatorComputationRepository.java

@@ -1,16 +1,29 @@
 package com.zenika.tech.lab.ingester.model;
 
+import java.util.List;
 import java.util.Map;
 import java.util.SortedSet;
+import java.util.stream.Collectors;
 
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+
+import com.zenika.tech.lab.ingester.Configuration;
 import com.zenika.tech.lab.ingester.model.export.ComputedIndicators.IndicatorDataPoint;
 
 import io.quarkus.hibernate.orm.panache.PanacheRepository;
 import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.Query;
 import jakarta.transaction.Transactional;
 
 @ApplicationScoped
 public class IndicatorComputationRepository implements PanacheRepository<IndicatorComputation>{
+	@ConfigProperty(name = Configuration.CONFIGURATION_PREFIX + "indicators.computations.findAllPriorized.sql")
+	public String findAllPriorized;
+	@Inject EntityManager entityManager;
+	private List<Long> priorizedIds;
 	@Transactional
 	public IndicatorComputation findOrCreateFromTechnology(Technology body, String indicatorRoute) {
 		IndicatorComputation returned = null;
@@ -26,4 +39,20 @@ public IndicatorComputation findOrCreateFromTechnology(Technology body, String i
 		persist(returned);
 		return returned;
 	}
+	
+	@Inject public void buildPriorizedIds(List<Technology> priorized) {
+		priorizedIds = priorized.stream()
+				.map(t -> t.id)
+				.collect(Collectors.toList());
+	}
+
+	public Iterable<IndicatorComputation> findAllPriorized() {
+		Query query = entityManager.createNativeQuery(findAllPriorized, IndicatorComputation.class);
+		query.setParameter("priorized", getPriorizedIds());
+		return query.getResultList();
+	}
+
+	List<Long> getPriorizedIds() {
+		return priorizedIds;
+	}
 }

src/main/java/com/zenika/tech/lab/ingester/model/PriorizedTechnologiesProducer.java

@@ -0,0 +1,51 @@
+package com.zenika.tech.lab.ingester.model;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.eclipse.microprofile.config.inject.ConfigProperty;
+
+import com.zenika.tech.lab.ingester.Configuration;
+
+import io.smallrye.common.annotation.Identifier;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Produces;
+import jakarta.inject.Named;
+import jakarta.transaction.Transactional;
+import io.quarkus.logging.Log;
+import io.quarkus.panache.common.Parameters;
+
+@ApplicationScoped
+public class PriorizedTechnologiesProducer {
+	@Produces
+	@Transactional
+	@Named("priorized")
+	List<Technology> buildPriorizedTechnologiesList(
+			@ConfigProperty(name=Configuration.CONFIGURATION_PREFIX + "priorized-technologies") List<String> priorized,
+			TechnologyRepository technologies) {
+		// In a very, VERY, strange fashion, config is not obtained from injection,
+		// but from that damn old ServiceProvider thingie
+		return priorized.stream()
+				.map(technologyIdentifier -> loadTechnology(technologyIdentifier, technologies))
+				.flatMap(Optional::stream)
+				.collect(Collectors.toList());
+	}
+
+	private Optional<Technology> loadTechnology(String value, TechnologyRepository technologies) {
+		String[] parts = value.split(":");
+		if(parts.length!=2) {
+			Log.errorf("configuration property tech-lab-ingester.priorized-technologies contains the invalid value \"\"", value);
+			return Optional.empty();
+		} else {
+			return loadTechnology(parts[0], parts[1], technologies);
+		}
+		
+	}
+
+	private Optional<Technology> loadTechnology(String platform, String name, TechnologyRepository technologies) {
+		return technologies.find("platform=:p and name=:n",
+				Parameters.with("p", platform).and("n", name)
+				).firstResultOptional();
+	}
+}

src/main/java/com/zenika/tech/lab/ingester/model/Technology.java

@@ -52,7 +52,7 @@ public class Technology extends PanacheEntityBase {
 	public String packageManagerUrl;
 	@AvroDoc("source code repository url")
 	public String repositoryUrl;
-	@AvroDoc("Distyribution platform for the technology. It doesn't implies a specific language is used.")
+	@AvroDoc("Distribution platform for the technology. It doesn't implies a specific language is used.")
 	public String platform;
 
 	public Technology() {}

src/main/java/com/zenika/tech/lab/ingester/processors/IndicatorComputationProcessor.java

@@ -3,37 +3,39 @@
 import java.util.Date;
 import java.util.List;
 
-import org.apache.camel.Exchange;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
 
+import com.zenika.tech.lab.ingester.Configuration;
 import com.zenika.tech.lab.ingester.model.IndicatorComputation;
+import com.zenika.tech.lab.ingester.model.IndicatorComputation.IndicatorComputationStatus;
 import com.zenika.tech.lab.ingester.model.IndicatorComputationRepository;
 import com.zenika.tech.lab.ingester.model.Technology;
-import com.zenika.tech.lab.ingester.model.IndicatorComputation.IndicatorComputationStatus;
-import com.zenika.tech.lab.ingester.model.export.ComputedIndicators;
 
 import io.quarkus.panache.common.Sort;
 import io.quarkus.panache.common.Sort.Direction;
 import jakarta.enterprise.context.Dependent;
 import jakarta.inject.Inject;
+import jakarta.inject.Named;
+import jakarta.persistence.Query;
 import jakarta.transaction.Transactional;
 
 @Dependent
 public class IndicatorComputationProcessor {
 	@Inject IndicatorComputationRepository indicators;
 
-	
 	public void generateIndicatorComputationFor(Technology technology, String r) {
 		indicators.findOrCreateFromTechnology(technology, r);
 	}
 
+	/**
+	 * Find all indicator computations sorted by priority.
+	 * This priority first takes in account the fact that technology is in 
+	 * the priorized technologies list, then sort by indicator computation date
+	 * @return
+	 */
 	@Transactional
-	public Iterable<IndicatorComputation> findAllOldestFirst() {
-		List<IndicatorComputation> returned = indicators.findAll(
-				// This ensure we first finish currently computing indicators prior to computing new ones
-				Sort.by("status", Direction.Descending)
-				.and("date", Direction.Ascending)
-				.and("id", Direction.Ascending)).list();
-		return returned;
+	public Iterable<IndicatorComputation> findAllPriorized() {
+		return indicators.findAllPriorized();
 	}
 
 	@Transactional

src/main/resources/application.yaml

@@ -3,14 +3,16 @@
     export:
       INDICATOR:
         csv:
+          enabled: true
           to:
             gcp:
-              enabled: false
+              enabled: true
       TECHNOLOGY:
         csv:
+          enabled: true
           to:
             gcp:
-              enabled: false
+              enabled: true
     projects:
       per:
         page: 10
@@ -88,6 +90,9 @@ quarkus:
         Content-Type: application/json; charset=utf-8
       url: https://api.github.com/graphql
 tech-lab-ingester:
+  # Values in this key should be in the form "platform:name"
+  priorized-technologies:
+    - NPM:react
   export:
     INDICATOR_COMPUTATION:
       csv:
@@ -104,6 +109,16 @@ tech-lab-ingester:
             enabled: false
 
   indicators:
+    computations:
+      findAllPriorized:
+        sql: |
+          select *
+            from indicator_computation ic
+            order by 
+            ic.technology_id in :priorized desc,
+            ic.status desc,
+            ic.indicator_date  asc,
+            ic.technology_id asc;
     github:
       stars:
         graphql:

src/test/java/com/zenika/tech/lab/ingester/ProcessIndicatorComputationsTest.java

@@ -9,14 +9,20 @@
 import org.apache.camel.quarkus.test.CamelQuarkusTestSupport;
 import org.assertj.core.api.Assertions;
 import org.assertj.core.api.InstanceOfAssertFactories;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 import com.zenika.tech.lab.ingester.model.IndicatorComputation;
 import com.zenika.tech.lab.ingester.model.IndicatorComputation.IndicatorComputationStatus;
 import com.zenika.tech.lab.ingester.model.IndicatorComputationBuilder;
+import com.zenika.tech.lab.ingester.model.IndicatorComputationRepository;
 import com.zenika.tech.lab.ingester.model.Technology;
 import com.zenika.tech.lab.ingester.model.TechnologyBuilder;
+import com.zenika.tech.lab.ingester.model.TechnologyRepository;
 
+import io.quarkus.hibernate.orm.panache.PanacheQuery;
+import io.quarkus.test.junit.QuarkusMock;
 import io.quarkus.test.junit.QuarkusTest;
 import jakarta.inject.Inject;
 
@@ -25,19 +31,33 @@ class ProcessIndicatorComputationsTest extends CamelQuarkusTestSupport {
 
 	@Inject	ProducerTemplate producerTemplate;
 	@Inject ConsumerTemplate consumerTemplate;
+	@Inject IndicatorComputationRepository indicatorsToMock;
 
+	static Technology TESTED_TECHNOLOGY = TechnologyBuilder.technology()
+			.build();
+	static String INDICATOR_ROUTE = "log:done with that fake indicator route";
+	static IndicatorComputation indicatorComputation = IndicatorComputationBuilder
+			.indicatorComputation()
+			.technology(TESTED_TECHNOLOGY)
+			.indicatorRoute(INDICATOR_ROUTE)
+			.build();
+
+	@BeforeAll
+	public static void setup() {
+		IndicatorComputationRepository mockingIndicators = Mockito.mock(IndicatorComputationRepository.class);
+		PanacheQuery mockedQuery = Mockito.mock(PanacheQuery.class);
+		Mockito.when(mockedQuery.firstResult()).thenReturn(indicatorComputation);
+		Mockito
+			.when(mockingIndicators.find(Mockito.anyString(), 
+					Mockito.<Object[]>any() ))
+			.thenReturn(mockedQuery);
+		QuarkusMock.installMockForType(mockingIndicators, IndicatorComputationRepository.class);
+	}
 
 	@Test
 	void testRoutingToCorrectComputer() throws Exception {
 		// Given
-		Technology TESTED_TECHNOLOGY = TechnologyBuilder.technology()
-				.build();
-		String INDICATOR_ROUTE = "NO INDICATOR ROUTE CONFIFURED";
-		IndicatorComputation indicatorComputation = IndicatorComputationBuilder
-				.indicatorComputation()
-				.technology(TESTED_TECHNOLOGY)
-				.indicatorRoute(INDICATOR_ROUTE)
-				.build();
+		
 		// Replace standard call (which would get data from database)
 		// by our call (which get only one indicator computation)
 		AdviceWith.adviceWith(context, ProcessIndicatorComputations.FETCH_ALL_INDICATOR_COMPUTATIONS_ROUTE_ID, a -> {