Merge pull request #11347 from IQSS/11280-search-api-count-per-object

ofahimIQSS · web-flow · commit fb33fa4f5b75 · 2025-03-25T12:27:56.000-04:00
return all type totals regardless of types being returned
diff --git a/doc/release-notes/11280-search-api-count-per-object.md b/doc/release-notes/11280-search-api-count-per-object.md
@@ -0,0 +1,7 @@
+## Bug fix to Search API. Now includes all type totals (Dataverses, Dataset, and Files) regardless of the list of types requested
+
+None requested types were returned with total count set to 0.
+&type=dataverse&type=dataset would result in "Files" : 0 since type=file was not requested
+
+Now all counts show the correct totals.
+Note: This is only true for the first page requested. Subsequent pages could have 0 counts and should not be used. This is due to the need for speed. Getting the totals is an additional search call in the background.
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.search.SearchFields;
 import edu.harvard.iq.dataverse.search.FacetCategory;
 import edu.harvard.iq.dataverse.search.FacetLabel;
@@ -17,10 +18,7 @@
 import edu.harvard.iq.dataverse.search.SortBy;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.logging.Logger;
 import jakarta.ejb.EJB;
 import jakarta.inject.Inject;
@@ -92,9 +90,50 @@ public Response search(
             String geoPoint;
             String geoRadius;
             List<Dataverse> dataverseSubtrees = new ArrayList<>();
+            DataverseRequest requestUser = createDataverseRequest(user);
+            String allTypes = ":(" + SearchConstants.DATAVERSES + " OR " + SearchConstants.DATASETS + " OR " + SearchConstants.FILES + ")";
+            Map<String, Long> objectTypeCountsMap = new HashMap<>(3);
+            objectTypeCountsMap.put(SearchConstants.UI_DATAVERSES, 0L);
+            objectTypeCountsMap.put(SearchConstants.UI_DATASETS, 0L);
+            objectTypeCountsMap.put(SearchConstants.UI_FILES, 0L);
+
+            // users can't change these (yet anyway)
+            boolean dataRelatedToMe = showMyData; //getDataRelatedToMe();
 
             try {
+                // we have to add "" (root) otherwise there is no permissions check
+                if (subtrees.isEmpty()) {
+                    dataverseSubtrees.add(getSubtree(""));
+                }
+                else {
+                    for (String subtree : subtrees) {
+                        dataverseSubtrees.add(getSubtree(subtree));
+                    }
+                }
+                filterQueries.add(getFilterQueryFromSubtrees(dataverseSubtrees));
+
                 if (!types.isEmpty()) {
+                    // Query to get the totals if needed.
+                    // Only needed if the list of types doesn't include all types since missing types will default to count of 0
+                    // Only get the totals for the first page (paginationStart == 0) for speed
+                    if (showTypeCounts && types.size() < objectTypeCountsMap.size() && paginationStart == 0) {
+                        List<String> totalFilterQueries = new ArrayList<>();
+                        totalFilterQueries.addAll(filterQueries);
+                        totalFilterQueries.add(SearchFields.TYPE + allTypes);
+                        try {
+                            SolrQueryResponse resp = searchService.search(requestUser, dataverseSubtrees, query, totalFilterQueries, null, null, 0,
+                                    dataRelatedToMe, 1, false, null, null, false, false);
+                            if (resp != null) {
+                                for (FacetCategory facetCategory : resp.getTypeFacetCategories()) {
+                                    for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                                        objectTypeCountsMap.put(facetLabel.getName(), facetLabel.getCount());
+                                    }
+                                }
+                            }
+                        } catch(Exception e) {
+                            logger.info("Search getting total counts: " + e.getMessage());
+                        }
+                    }
                     filterQueries.add(getFilterQueryFromTypes(types));
                 } else {
                     /**
@@ -103,22 +142,11 @@ public Response search(
                      * SearchServiceBean tries to get SearchFields.TYPE. The GUI
                      * always seems to add SearchFields.TYPE, even for superusers.
                      */
-                    filterQueries.add(SearchFields.TYPE + ":(" + SearchConstants.DATAVERSES + " OR " + SearchConstants.DATASETS + " OR " + SearchConstants.FILES + ")");
+                    filterQueries.add(SearchFields.TYPE + allTypes);
                 }
                 sortBy = SearchUtil.getSortBy(sortField, sortOrder);
                 numResultsPerPage = getNumberOfResultsPerPage(numResultsPerPageRequested);
                 
-                 // we have to add "" (root) otherwise there is no permissions check
-                if(subtrees.isEmpty()) {
-                    dataverseSubtrees.add(getSubtree(""));
-                }
-                else {
-                    for(String subtree : subtrees) {
-                        dataverseSubtrees.add(getSubtree(subtree));
-                    }
-                }
-                filterQueries.add(getFilterQueryFromSubtrees(dataverseSubtrees));
-                
                 if(filterQueries.isEmpty()) { //Extra sanity check just in case someone else touches this
                     throw new IOException("Filter is empty, which should never happen, as this allows unfettered searching of our index");
                 }
@@ -137,13 +165,10 @@ public Response search(
             } catch (Exception ex) {
                 return error(Response.Status.BAD_REQUEST, ex.getLocalizedMessage());
             }
-
-            // users can't change these (yet anyway)
-            boolean dataRelatedToMe = showMyData; //getDataRelatedToMe();
             
             SolrQueryResponse solrQueryResponse;
             try {
-                solrQueryResponse = searchService.search(createDataverseRequest(user),
+                solrQueryResponse = searchService.search(requestUser,
                         dataverseSubtrees,
                         query,
                         filterQueries,
@@ -211,50 +236,17 @@ public Response search(
             }
 
             value.add("count_in_response", solrSearchResults.size());
-            
-            // we want to show the missing dvobject types with count = 0
-            // per https://github.com/IQSS/dataverse/issues/11127
 
             if (showTypeCounts) {
-                JsonObjectBuilder objectTypeCounts = Json.createObjectBuilder();
-                if (!solrQueryResponse.getTypeFacetCategories().isEmpty()) {
-                    boolean filesMissing = true;
-                    boolean datasetsMissing = true;
-                    boolean dataversesMissing = true;
-                    for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) {
-                        for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
-                            objectTypeCounts.add(facetLabel.getName(), facetLabel.getCount());
-                            if (facetLabel.getName().equals((SearchConstants.UI_DATAVERSES))) {
-                                dataversesMissing = false;
-                            }
-                            if (facetLabel.getName().equals((SearchConstants.UI_DATASETS))) {
-                                datasetsMissing = false;
-                            }
-                            if (facetLabel.getName().equals((SearchConstants.UI_FILES))) {
-                                filesMissing = false;
-                            }
-                        }
-                    }
-
-                    if (solrQueryResponse.getTypeFacetCategories().size() < 3) {
-                        if (dataversesMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0);
-                        }
-                        if (datasetsMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_DATASETS, 0);
-                        }
-                        if (filesMissing) {
-                            objectTypeCounts.add(SearchConstants.UI_FILES, 0);
+                for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) {
+                    for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                        if (facetLabel.getCount() > 0) {
+                            objectTypeCountsMap.put(facetLabel.getName(), facetLabel.getCount());
                         }
                     }
-
-                }
-                if (showTypeCounts && solrQueryResponse.getTypeFacetCategories().isEmpty()) {
-                    objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0);
-                    objectTypeCounts.add(SearchConstants.UI_DATASETS, 0);
-                    objectTypeCounts.add(SearchConstants.UI_FILES, 0);
                 }
-
+                JsonObjectBuilder objectTypeCounts = Json.createObjectBuilder();
+                objectTypeCountsMap.forEach((k,v) -> objectTypeCounts.add(k,v));
                 value.add("total_count_per_object_type", objectTypeCounts);
             }
             /**
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java
@@ -1767,6 +1767,7 @@ public void testShowTypeCounts() throws InterruptedException  {
         Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken, affiliation);
         assertEquals(201, createDataverseResponse.getStatusCode());
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+        String dataverseWithDatasetsFilesAlias = dataverseAlias;
 
         // create 3 Datasets, each with 2 Datafiles
         for (int i = 0; i < 3; i++) {
@@ -1840,7 +1841,35 @@ public void testShowTypeCounts() throws InterruptedException  {
                 .statusCode(OK.getStatusCode())
                 .body("data.total_count_per_object_type.Dataverses", CoreMatchers.is(1))
                 .body("data.total_count_per_object_type.Datasets", CoreMatchers.is(0))
-                .body("data.total_count_per_object_type.Files", CoreMatchers.is(0));        
+                .body("data.total_count_per_object_type.Files", CoreMatchers.is(0));
+
+        // Test Search with show_type_counts = TRUE getting only Dataverses
+        searchResp = UtilIT.search(dataverseWithDatasetsFilesAlias, apiToken, "&show_facets=true&type=dataverse&show_type_counts=true");
+        searchResp.prettyPrint();
+        searchResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.count_in_response", CoreMatchers.is(1)) // 1 dataverse
+                .body("data.total_count_per_object_type.Dataverses", CoreMatchers.is(1))
+                .body("data.total_count_per_object_type.Datasets", CoreMatchers.is(3))
+                .body("data.total_count_per_object_type.Files", CoreMatchers.is(6));
+        // Test Search with show_type_counts = TRUE getting only Dataverses and Datasets
+        searchResp = UtilIT.search(dataverseWithDatasetsFilesAlias, apiToken, "&type=dataverse&type=dataset&show_type_counts=true");
+        searchResp.prettyPrint();
+        searchResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.count_in_response", CoreMatchers.is(4)) // 1 dataverse + 3 datasets
+                .body("data.total_count_per_object_type.Dataverses", CoreMatchers.is(1))
+                .body("data.total_count_per_object_type.Datasets", CoreMatchers.is(3))
+                .body("data.total_count_per_object_type.Files", CoreMatchers.is(6));
+        // Test Search with show_type_counts = TRUE getting Dataverses, Datasets and Files
+        searchResp = UtilIT.search(dataverseWithDatasetsFilesAlias, apiToken, "&type=dataverse&type=dataset&type=file&show_type_counts=true");
+        searchResp.prettyPrint();
+        searchResp.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.count_in_response", CoreMatchers.is(10)) // 1 dataverse + 3 datasets + 6 files
+                .body("data.total_count_per_object_type.Dataverses", CoreMatchers.is(1))
+                .body("data.total_count_per_object_type.Datasets", CoreMatchers.is(3))
+                .body("data.total_count_per_object_type.Files", CoreMatchers.is(6));
     }
 
     @Test
@@ -1891,4 +1920,47 @@ public void testTabularFiles() throws IOException {
                 .body("data.items[0].observations", is(3));
     }
 
+    @Test
+    public void testTotalCount() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        Response makeSuperUser = UtilIT.makeSuperUser(username);
+        assertEquals(200, makeSuperUser.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        createDataverseResponse.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDataset.prettyPrint();
+        createDataset.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset);
+        String datasetPid = UtilIT.getDatasetPersistentIdFromResponse(createDataset);
+
+        Path pathToDataFile = Paths.get(java.nio.file.Files.createTempDirectory(null) + File.separator + "data.csv");
+        String contentOfCsv = ""
+                + "name,pounds,species,treats\n"
+                + "Midnight,15,dog,milkbones\n"
+                + "Tiger,17,cat,cat grass\n"
+                + "Panther,21,cat,cat nip\n";
+        java.nio.file.Files.write(pathToDataFile, contentOfCsv.getBytes());
+
+        Response uploadFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToDataFile.toString(), apiToken);
+        uploadFile.prettyPrint();
+        uploadFile.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.files[0].label", equalTo("data.csv"));
+
+        Response searchResponse = UtilIT.search("&show_facets=true&sort=date&order=desc&show_type_counts=true&subtree=root&per_page=10&type=dataverse&type=dataset", apiToken);
+        searchResponse.prettyPrint();
+        searchResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+    }
 }
