added fix for prometheus tests (#193)

Author: Aman Brar

sample-apps/prometheus/main.go

@@ -74,9 +74,9 @@ func loadMetricCountFromEnv() int {
 func updateMetrics() {
 	for {
 		time.Sleep(time.Second * 30)
+		mtx.Lock()
 		mc.timestamp = float64(time.Now().UnixNano()) / 1000000000
 		for idx := 0; idx < mc.metricCount; idx++ {
-			mtx.Lock()
 			mc.counters[idx].Add(rand.Float64())
 			mc.gauges[idx].Add(rand.Float64())
 			lowerBound := math.Mod(rand.Float64(), 1)
@@ -85,8 +85,8 @@ func updateMetrics() {
 				mc.histograms[idx].Observe(i)
 				mc.summarys[idx].Observe(i)
 			}
-			mtx.Unlock()
 		}
+		mtx.Unlock()
 	}
 }
 

terraform/eks/pull_mode_deployment.tf

@@ -2,15 +2,9 @@
 # Pull mode deployments
 ##########################################
 
-resource "time_sleep" "wait_until_sample_app_produces_metrics" {
-  create_duration = "30s"
-  depends_on = [kubernetes_service.sample_app_service]
-}
-
 # deploy aoc and mocked server
 resource "kubernetes_deployment" "pull_mode_aoc_deployment" {
   count = var.sample_app_mode == "pull" ? 1 : 0
-  depends_on = [time_sleep.wait_until_sample_app_produces_metrics]
 
   metadata {
     name = "aoc"

validator/src/main/java/com/amazon/aoc/models/prometheus/PrometheusMetric.java

@@ -78,7 +78,7 @@ public static int staticPrometheusMetricCompare(PrometheusMetric o1, PrometheusM
       }
 
       // check metric value
-      if (o1.getMetricValue().equals(o2.getMetricValue())) {
+      if (!o1.getMetricValue().equals(o2.getMetricValue())) {
         return o1.getMetricValue().compareTo(o2.getMetricValue());
       }
 

validator/src/main/java/com/amazon/aoc/validators/PrometheusMetricValidator.java

@@ -50,15 +50,19 @@ public class PrometheusMetricValidator implements IValidator {
   @Override
   public void validate() throws Exception {
     log.info("Start prometheus metric validating");
-    log.info("sleeping to allow metrics to propogate through AOC");
-    TimeUnit.SECONDS.sleep(30);
+    log.info("allow sample app load balancer to start");
+    TimeUnit.SECONDS.sleep(60);
     log.info("resuming validation");
 
     RetryHelper.retry(
         MAX_RETRY_COUNT,
         () -> {
           // get expected metrics
           List<PrometheusMetric> expectedMetricList = this.getExpectedMetricList(context);
+
+          // Since we add 20s to timestamp, must sleep 20s between requests
+          TimeUnit.SECONDS.sleep(20);
+
           // get metric from cortex
           CortexService cortexService = new CortexService(context);
 

validator/src/main/resources/validations/prometheus-metric-validation.yml

@@ -1,4 +1,4 @@
 -
   validationType: "prom-metric"
-  shouldValidateMetricValue: true
+  shouldValidateMetricValue: false
   expectedResultPath: "/expected_metrics"