Merge pull request #11189 from GlobalDataverseCommunityConsortium/Metrics-fix_file/monthly

File metrics fixes

Author: ofahimIQSS

doc/release-notes/Metrics-FixFileAPIs.md

@@ -0,0 +1,2 @@
+The /api/info/metrics/files/monthly API call had a bug that resulted in files being counted each time they were published in a new version if those publication events occurred in different months. This resulted in an over-count.
+The /api/info/metrics/files and /api/info/metrics/files/toMonth API calls had a bug that resulted in files that were published but no longer in the latest published version as of the specified date (now, or the date entered in the /toMonth variant). This resulted in an under-count.

src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java

@@ -288,11 +288,18 @@ public JsonArray filesTimeSeries(Dataverse d) {
                         + "from (\n"
                         + "select min(to_char(COALESCE(releasetime, createtime), 'YYYY-MM')) as date, filemetadata.id as id\n"
                         + "from datasetversion, filemetadata\n"
-                        + "where datasetversion.id=filemetadata.datasetversion_id\n"
-                        + "and versionstate='RELEASED' \n"
-                        + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n"
+                        + "where datasetversion.id = filemetadata.datasetversion_id\n"
+                        + "and datasetversion.versionstate = 'RELEASED'\n"
+                        + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id = dvobject.id\n"
                         + "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n "
                         + ((d == null) ? ")" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + "))\n ")
+                        + "and filemetadata.id = (\n"
+                        + "    select min(fm.id)\n"
+                        + "    from filemetadata fm\n"
+                        + "    join datasetversion dv on dv.id = fm.datasetversion_id\n"
+                        + "    where fm.datafile_id = filemetadata.datafile_id\n"
+                        + "    and dv.versionstate = 'RELEASED'\n"
+                        + ")\n"
                         + "group by filemetadata.id) as subq group by subq.date order by date;");
         logger.log(Level.FINE, "Metric query: {0}", query);
         List<Object[]> results = query.getResultList();
@@ -314,8 +321,9 @@ public long filesToMonth(String yyyymm, Dataverse d) {
                 + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
+                + "join filemetadata fm on fm.datasetversion_id = datasetversion.id\n"
                 + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
-                + "where versionstate='RELEASED'\n"
+                + "where datasetversion.versionstate='RELEASED' and filemetadata.datafile_id=fm.datafile_id\n"
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + "and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
                 + "and dataset.harvestingclient_id is null\n"
@@ -353,12 +361,14 @@ public long filesPastDays(int days, Dataverse d) {
 
     public JsonArray filesByType(Dataverse d) {
         // SELECT DISTINCT df.contenttype, sum(df.filesize) FROM datafile df, dvObject ob where ob.id = df.id and dob.owner_id< group by df.contenttype
-        // ToDo - published only?
         Query query = em.createNativeQuery("SELECT DISTINCT df.contenttype, count(df.id), coalesce(sum(df.filesize), 0) "
-                + " FROM DataFile df, DvObject ob"
-                + " where ob.id = df.id "
-                + ((d == null) ? "" : "and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n")
-                + "group by df.contenttype;");
+                + " FROM DataFile df "
+                + " JOIN DvObject ob ON ob.id = df.id "
+                + " JOIN FileMetadata fm ON fm.datafile_id = df.id "
+                + " JOIN DatasetVersion dv ON dv.id = fm.datasetversion_id "
+                + " WHERE dv.versionstate = 'RELEASED' "
+                + ((d == null) ? "" : "AND ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ") ")
+                + "GROUP BY df.contenttype;");
         JsonArrayBuilder jab = Json.createArrayBuilder();
         try {
             List<Object[]> results = query.getResultList();