Merge pull request #33 from OpenSRP/issue/15-disable-incremental-counts

Disable incremental counts & add support for returning 0 in multi-result queries

Author: ekigamba

.travis.yml

@@ -14,14 +14,6 @@ cache:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
 
-env:
-  matrix:
-    - ANDROID_TARGET=android-22 ANDROID_ABI=armeabi-v7a
-  global:
-    # wait up to 10 minutes for adb to connect to emulator
-    - MALLOC_ARENA_MAX=2
-    - ADB_INSTALL_TIMEOUT=20 #increment timeout to 20 mins
-
 android:
   components:
     # tools required
@@ -35,17 +27,6 @@ android:
     - extra-google-m2repository
     - extra-android-m2repository
 
-    # Specify at least one system image,
-    # if you need to run emulator(s) during your tests
-    - sys-img-armeabi-v7a-android-22
-
-before_script:
-  # Emulator Management: Create, Start and Wait
-  - echo no | android create avd --force -n test -t $ANDROID_TARGET --abi $ANDROID_ABI
-  - emulator -avd test -no-skin -no-audio -no-window &
-  - android-wait-for-emulator
-  - adb shell input keyevent 82 &
-
 script:
   - echo "Travis branch is $TRAVIS_BRANCH"
   - echo "Travis branch is in pull request $TRAVIS_PULL+REQUEST"

gradle.properties

@@ -1,4 +1,4 @@
-VERSION_NAME=0.0.8-SNAPSHOT
+VERSION_NAME=0.0.9-SNAPSHOT
 VERSION_CODE=1
 GROUP=org.smartregister
 POM_SETTING_DESCRIPTION=OpenSRP Client Reporting Library

opensrp-reporting/src/androidTest/java/org/smartregister/reporting/ExampleInstrumentedTest.java

@@ -226,9 +226,11 @@ public void readConfigFile(String configFilePath, SQLiteDatabase sqLiteDatabase)
                         indicatorQuery = new IndicatorQuery(null, indicatorYamlConfigItem.getKey()
                                 , indicatorYamlConfigItem.getIndicatorQuery()
                                 , 0
-                                , indicatorYamlConfigItem.isMultiResult());
+                                , indicatorYamlConfigItem.isMultiResult()
+                                , indicatorYamlConfigItem.getExpectedIndicators());
                         indicatorQueries.add(indicatorQuery);
                     }
+
                     reportIndicators.add(indicator);
                 }
             }

opensrp-reporting/src/main/java/org/smartregister/reporting/ReportingLibrary.java

@@ -2,6 +2,8 @@
 
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
+import android.support.annotation.VisibleForTesting;
+import android.text.TextUtils;
 
 import com.google.gson.Gson;
 
@@ -16,12 +18,12 @@
 import org.smartregister.reporting.repository.DailyIndicatorCountRepository;
 import org.smartregister.reporting.repository.IndicatorQueryRepository;
 import org.smartregister.reporting.repository.IndicatorRepository;
-import org.smartregister.reporting.util.AppProperties;
 import org.smartregister.repository.EventClientRepository;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -39,12 +41,12 @@
  */
 
 public class ReportIndicatorDaoImpl implements ReportIndicatorDao {
-
     public static final String REPORT_LAST_PROCESSED_DATE = "REPORT_LAST_PROCESSED_DATE";
+    public static String DAILY_TALLY_DATE_FORMAT = "yyyy-MM-dd";
+
     public static String PREVIOUS_REPORT_DATES_QUERY = "select distinct eventDate, " + EventClientRepository.event_column.updatedAt + " from "
             + EventClientRepository.Table.event.name();
 
-    private static String TAG = ReportIndicatorDaoImpl.class.getCanonicalName();
     private static String eventDateFormat = "yyyy-MM-dd HH:mm:ss";
     private IndicatorQueryRepository indicatorQueryRepository;
     private DailyIndicatorCountRepository dailyIndicatorCountRepository;
@@ -80,13 +82,19 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) {
         float count;
         SQLiteDatabase database = ReportingLibrary.getInstance().getRepository().getWritableDatabase();
 
-        LinkedHashMap<String, Date> reportEventDates = getReportEventDates(lastProcessedDate, database);
+        Date timeNow = Calendar.getInstance().getTime();
+        LinkedHashMap<String, Date> reportEventDates = getReportEventDates(timeNow, lastProcessedDate, database);
+
         Map<String, IndicatorQuery> indicatorQueries = indicatorQueryRepository.getAllIndicatorQueries();
 
         if (!reportEventDates.isEmpty() && !indicatorQueries.isEmpty()) {
-            String lastUpdatedDate = "";
+            String lastUpdatedDate = null;
+
             for (Map.Entry<String, Date> dates : reportEventDates.entrySet()) {
-                lastUpdatedDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(dates.getValue());
+                if (dates.getValue().getTime() != timeNow.getTime()) {
+                    lastUpdatedDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(dates.getValue());
+                }
+
                 for (Map.Entry<String, IndicatorQuery> entry : indicatorQueries.entrySet()) {
                     IndicatorQuery indicatorQuery = entry.getValue();
                     CompositeIndicatorTally tally = null;
@@ -95,10 +103,11 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) {
                         ArrayList<Object> result = executeQueryAndReturnMultiResult(indicatorQuery.getQuery(), dates.getKey(), database);
 
                         // If the size contains actual result other than the column names which are at index 0
-                        if (result.size() > 1) {
+                        if (result.size() > 1 || (indicatorQuery.getExpectedIndicators() != null && indicatorQuery.getExpectedIndicators().size() > 0)) {
                             tally = new CompositeIndicatorTally();
-                            tally.setValueSet(new Gson().toJson(result));
                             tally.setValueSetFlag(true);
+                            tally.setValueSet(new Gson().toJson(result));
+                            tally.setExpectedIndicators(indicatorQuery.getExpectedIndicators());
                         }
                     } else {
                         count = executeQueryAndReturnCount(indicatorQuery.getQuery(), dates.getKey(), database);
@@ -122,12 +131,17 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) {
                 }
             }
 
-            ReportingLibrary.getInstance().getContext().allSharedPreferences().savePreference(REPORT_LAST_PROCESSED_DATE, lastUpdatedDate);
+            if (!TextUtils.isEmpty(lastUpdatedDate)) {
+                ReportingLibrary.getInstance().getContext().allSharedPreferences().savePreference(REPORT_LAST_PROCESSED_DATE, lastUpdatedDate);
+            }
+
             Timber.i("generateDailyIndicatorTallies: Generate daily tallies complete");
         }
     }
 
-    private LinkedHashMap<String, Date> getReportEventDates(String lastProcessedDate, SQLiteDatabase database) {
+    @VisibleForTesting
+    @NonNull
+    protected LinkedHashMap<String, Date> getReportEventDates(@NonNull Date timeNow, @Nullable String lastProcessedDate, @NonNull SQLiteDatabase database) {
 
         ArrayList<HashMap<String, String>> values;
         if (lastProcessedDate == null || lastProcessedDate.isEmpty()) {
@@ -136,37 +150,39 @@ private LinkedHashMap<String, Date> getReportEventDates(String lastProcessedDate
             values = dailyIndicatorCountRepository.rawQuery(database, PREVIOUS_REPORT_DATES_QUERY.concat(" where " + EventClientRepository.event_column.updatedAt + " > '" + lastProcessedDate + "'" + " order by eventDate asc"));
         }
 
-        LinkedHashMap<String, Date> result = new LinkedHashMap<>();
+        LinkedHashMap<String, Date> reportEventDates = new LinkedHashMap<>();
 
         Date eventDate;
         Date updateDate;
         for (HashMap<String, String> val : values) {
             eventDate = formatDate(val.get(EventClientRepository.event_column.eventDate.name()), eventDateFormat);
             updateDate = formatDate(val.get(EventClientRepository.event_column.updatedAt.name()), eventDateFormat);
 
-            String keyDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(eventDate);
+            String keyDate = new SimpleDateFormat(DAILY_TALLY_DATE_FORMAT, Locale.getDefault()).format(eventDate);
 
-            if (result.get(keyDate) != null && updateDate != null) {
-                if (result.get(keyDate).getTime() < updateDate.getTime()) {
-                    result.put(keyDate, updateDate);
+            if (reportEventDates.get(keyDate) != null && updateDate != null) {
+                if (reportEventDates.get(keyDate).getTime() < updateDate.getTime()) {
+                    reportEventDates.put(keyDate, updateDate);
                 }
             } else {
-                result.put(keyDate, updateDate);
+                reportEventDates.put(keyDate, updateDate);
             }
         }
-        return result;
+
+        String dateToday = new SimpleDateFormat(DAILY_TALLY_DATE_FORMAT, Locale.getDefault()).format(timeNow);
+
+        if (reportEventDates.get(dateToday) == null) {
+            reportEventDates.put(dateToday, timeNow);
+        }
+
+        return reportEventDates;
     }
 
     private float executeQueryAndReturnCount(String queryString, String date, SQLiteDatabase database) {
         // Use date in querying if specified
         String query = "";
         if (date != null) {
-            if(!ReportingLibrary.getInstance().getAppProperties().hasProperty(AppProperties.KEY.COUNT_INCREMENTAL)
-                    || ReportingLibrary.getInstance().getAppProperties().getPropertyBoolean(AppProperties.KEY.COUNT_INCREMENTAL)) {
-                query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString;
-            } else {
-                query = queryString.contains("%s") ? queryString.replaceAll("%s", date.split(" ")[0]) : queryString;
-            }
+            query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString;
             Timber.i("QUERY : %s", query);
         }
 
@@ -198,12 +214,13 @@ private float executeQueryAndReturnCount(String queryString, String date, SQLite
         return count;
     }
 
+    @NonNull
     private ArrayList<Object> executeQueryAndReturnMultiResult(@NonNull String queryString, @Nullable String date, @NonNull SQLiteDatabase database) {
         // Use date in querying if specified
         String query = "";
         if (date != null) {
             Timber.i("QUERY : %s", queryString);
-            query = queryString.contains("'%s'") ? String.format(queryString, date) : queryString;
+            query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString;
         }
         Cursor cursor = null;
         ArrayList<Object> rows = new ArrayList<>();
@@ -254,6 +271,7 @@ private ArrayList<Object> executeQueryAndReturnMultiResult(@NonNull String query
         return rows;
     }
 
+    @Nullable
     private Date formatDate(String date, String format) {
         try {
             return new SimpleDateFormat(format, Locale.getDefault()).parse(date);

opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java

@@ -1,8 +1,10 @@
 package org.smartregister.reporting.domain;
 
 import android.support.annotation.NonNull;
+import android.support.annotation.Nullable;
 
 import java.util.Date;
+import java.util.List;
 
 /**
  * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-07-10
@@ -13,6 +15,9 @@ public class CompositeIndicatorTally extends IndicatorTally {
     private String valueSet;
     private boolean isValueSet;
 
+    @Nullable
+    private List<String> expectedIndicators;
+
     public CompositeIndicatorTally() {
     }
 
@@ -42,4 +47,13 @@ public boolean isValueSet() {
     public void setValueSetFlag(boolean valueSet) {
         isValueSet = valueSet;
     }
+
+    @Nullable
+    public List<String> getExpectedIndicators() {
+        return expectedIndicators;
+    }
+
+    public void setExpectedIndicators(@Nullable List<String> expectedIndicators) {
+        this.expectedIndicators = expectedIndicators;
+    }
 }

opensrp-reporting/src/main/java/org/smartregister/reporting/domain/CompositeIndicatorTally.java

@@ -1,18 +1,24 @@
 package org.smartregister.reporting.domain;
 
+import android.support.annotation.Nullable;
+
+import java.util.List;
+
 public class IndicatorQuery {
     private Long id;
     private String indicatorCode;
     private String query;
     private int dbVersion;
     private boolean isMultiResult;
+    private List<String> expectedIndicators;
 
-    public IndicatorQuery(Long id, String indicatorCode, String query, int dbVersion, boolean isMultiResult) {
+    public IndicatorQuery(Long id, String indicatorCode, String query, int dbVersion, boolean isMultiResult, @Nullable List<String> expectedIndicators) {
         this.id = id;
         this.indicatorCode = indicatorCode;
         this.query = query;
         this.dbVersion = dbVersion;
         this.isMultiResult = isMultiResult;
+        this.expectedIndicators = expectedIndicators;
     }
 
     public IndicatorQuery() {
@@ -57,4 +63,12 @@ public boolean isMultiResult() {
     public void setMultiResult(boolean multiResult) {
         isMultiResult = multiResult;
     }
+
+    public List<String> getExpectedIndicators() {
+        return expectedIndicators;
+    }
+
+    public void setExpectedIndicators(List<String> expectedIndicators) {
+        this.expectedIndicators = expectedIndicators;
+    }
 }

opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorQuery.java

@@ -1,5 +1,7 @@
 package org.smartregister.reporting.domain;
 
+import java.util.List;
+
 public class IndicatorYamlConfigItem {
 
     public static String INDICATOR_PROPERTY = "indicatorData";
@@ -8,6 +10,8 @@ public class IndicatorYamlConfigItem {
     private String description;
     private String indicatorQuery;
     private boolean isMultiResult;
+    private List<String> expectedIndicators;
+
 
     public String getKey() {
         return key;
@@ -40,4 +44,12 @@ public boolean isMultiResult() {
     public void setMultiResult(boolean multiResult) {
         isMultiResult = multiResult;
     }
+
+    public List<String> getExpectedIndicators() {
+        return expectedIndicators;
+    }
+
+    public void setExpectedIndicators(List<String> expectedIndicators) {
+        this.expectedIndicators = expectedIndicators;
+    }
 }