Merge remote-tracking branch 'ncsa/develop' into docker-minio

Author: robkooper

CHANGELOG.md

@@ -11,17 +11,36 @@ of these changes a reindex of Elasticsearch is required. This can be started by
 ### Added
 - Can now switch storage provider to for example S3 storage. See also env.example for configuration options.
 - Added ability to delete extractor, both from api and GUI.
+- Script to test extractions through the API.
   [CATS-1044](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1044)
 - API add tags endpoint now returns the added tags.
   [CATS-1053](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1053)
 - Ability to search by creator name and email address for all resources.
 - List Spaces/Datasets/Collections created by each user on their User Profile.
   [CATS-1056](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1056)
-
+- Allow user to easily flip through the files in a dataset.
+  [CATS-1058](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1058)
+- Ability to filter the files and folders in a dataset when sorting is enabled.
+- Visualize existing relations between datasets on the dataset page. This can be extended other resource types.
+  [CATS-1000](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1000)
+  
 ### Fixed
 - When adding tags to a section of an image, show the new tag without having to refresh the page.
   [CATS-1053](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1053)
 
+### Changed
+- Removed buttons to remove datasets from spaces and collections from certain pages where it was creating confusion for 
+  the user. Users thought it was a dataset delete button instead. Also moved Remove button for subcollections to right 
+  side of page to be consistent with other widgets.
+  [CATS-1055](https://opensource.ncsa.illinois.edu/jira/browse/CATS-1055)
+
+## 1.8.4 - 2020-05-15
+**_Warning:_ This update modifies how information is stored in Elasticsearch for text based searching. To take advantage 
+of these changes a reindex of Elasticsearch is required. This can be started by an admin either from GUI or through the API.**
+
+### Fixed
+- Fixed a bug related to improper indexing of files in nested subfolders, which could also affect searching by parent dataset.
+
 ## 1.8.3 - 2020-04-28
 **_Warning:_ This update modifies how information is stored in Elasticsearch for text based searching. To take advantage 
 of these changes a reindex of Elasticsearch is required. This can be started by an admin either from GUI or through the API.**

app/controllers/Datasets.scala

@@ -483,8 +483,7 @@ class Datasets @Inject() (
   /**
    * Dataset.
    */
-  def dataset(id: UUID, currentSpace: Option[String], limit: Int) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
-
+  def dataset(id: UUID, currentSpace: Option[String], limit: Int, filter: Option[String]) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
     implicit val user = request.user
     Previewers.findDatasetPreviewers().foreach(p => Logger.debug("Previewer found " + p.id))
     datasets.get(id) match {
@@ -612,11 +611,16 @@ class Datasets @Inject() (
         // increment view count for dataset
         val view_data = datasets.incrementViews(id, user)
 
+        // related datasets
+        val relatedThings = relations.findRelationships(dataset.id.stringify, ResourceType.dataset, ResourceType.dataset)
+        val relatedDatasets = for(r <- relatedThings) yield NodeDataset(datasets.get(UUID(r.target.id)).get, r.rdfType)
+
+
         // view_data is passed as tuple in dataset case only, because template is at limit of 22 parameters
         Ok(views.html.dataset(dataset, commentsByDataset, filteredPreviewers.toList, m,
           decodedCollectionsInside.toList, sensors, Some(decodedSpaces_canRemove), toPublish, curPubObjects,
           currentSpace, limit, showDownload, accessData, canAddDatasetToCollection,
-          stagingAreaDefined, view_data, extractionGroups))
+          stagingAreaDefined, view_data, extractionGroups, relatedDatasets, filter))
       }
       case None => {
         Logger.error("Error getting dataset" + id)
@@ -625,78 +629,74 @@ class Datasets @Inject() (
     }
   }
 
-  def getUpdatedFilesAndFolders(datasetId: UUID, limit: Int, pageIndex: Int, space: Option[String]) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, datasetId)))(parse.json) { implicit request =>
+  def getUpdatedFilesAndFolders(datasetId: UUID, limit: Int, pageIndex: Int, space: Option[String], filter: Option[String]) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, datasetId)))(parse.json) { implicit request =>
     implicit val user = request.user
     val filepageUpdate = if (pageIndex < 0) 0 else pageIndex
     val sortOrder: String =
       request.cookies.get("sort-order") match {
         case Some(cookie) => cookie.value
         case None => "dateN" //If there is no cookie, and an order was not passed in, the view will choose its default
       }
+
     datasets.get(datasetId) match {
       case Some(dataset) => {
+        val folderHierarchy = new ListBuffer[Folder]()
+
         val folderId = (request.body \ "folderId").asOpt[String]
-        folderId match {
-          case Some(fId) => {
-            folders.get(UUID(fId)) match {
+        val (childFolders: List[UUID], childFiles: List[UUID]) =
+          folderId match {
+            case Some(fId) => folders.get(UUID(fId)) match {
               case Some(folder) => {
-                val (foldersList: List[Folder], limitFileList: List[File]) =
-                  if(play.Play.application().configuration().getBoolean("sortInMemory")) {
-                    (SortingUtils.sortFolders(folder.folders.flatMap(f => folders.get(f)), sortOrder).slice(limit * filepageUpdate, limit * (filepageUpdate + 1)),
-                     SortingUtils.sortFiles(files.get(folder.files).found, sortOrder).slice(limit * filepageUpdate - folder.folders.length, limit * (filepageUpdate + 1) - folder.folders.length))
-                  } else {
-                    (folder.folders.reverse.slice(limit * filepageUpdate, limit * (filepageUpdate+1)).flatMap(f => folders.get(f)),
-                     folder.files.reverse.slice(limit * filepageUpdate - folder.folders.length, limit * (filepageUpdate+1) - folder.folders.length).flatMap(f => files.get(f)))
-                  }
-                var folderHierarchy = new ListBuffer[Folder]()
                 folderHierarchy += folder
-                var f1: Folder = folder
+                var f1 = folder
                 while (f1.parentType == "folder") {
                   folders.get(f1.parentId) match {
                     case Some(fparent) => {
                       folderHierarchy += fparent
                       f1 = fparent
                     }
-                    case None =>
+                    case None => Logger.error("Parent folder " + f1.parentId.toString + " not found.")
                   }
                 }
-                val fileComments = limitFileList.map { file =>
-                  var allComments = comments.findCommentsByFileId(file.id)
-                  sections.findByFileId(file.id).map { section =>
-                    allComments ++= comments.findCommentsBySectionId(section.id)
-                  }
-                  file.id -> allComments.size
-                }.toMap
-                val next = folder.files.length + folder.folders.length > limit * (filepageUpdate + 1)
-
-                Ok(views.html.datasets.filesAndFolders(dataset, Some(folder.id.stringify), foldersList, folderHierarchy.reverse.toList, pageIndex, next, limitFileList.toList, fileComments, space)(request.user))
-
+                (folder.folders, folder.files)
               }
-              case None => InternalServerError(s"No folder with id $fId found")
+              case None => Logger.error("Folder " + fId + " not found.")
             }
+            case None => (dataset.folders, dataset.files)
           }
-          case None => {
-            val (foldersList: List[Folder], limitFileList: List[File]) = if(play.Play.application().configuration().getBoolean("sortInMemory")) {
-              (SortingUtils.sortFolders(dataset.folders.flatMap(f => folders.get(f)), sortOrder).slice(limit * filepageUpdate, limit * (filepageUpdate + 1)),
-               SortingUtils.sortFiles(files.get(dataset.files).found, sortOrder).slice(limit * filepageUpdate - dataset.folders.length, limit * (filepageUpdate + 1) - dataset.folders.length))
-            } else {
-              (dataset.folders.reverse.slice(limit * filepageUpdate, limit * (filepageUpdate+1)).flatMap(f => folders.get(f)),
-               dataset.files.reverse.slice(limit * filepageUpdate - dataset.folders.length, limit * (filepageUpdate+1) - dataset.folders.length).flatMap(f => files.get(f)))
-            }
 
-            val fileComments = limitFileList.map { file =>
-              var allComments = comments.findCommentsByFileId(file.id)
-              sections.findByFileId(file.id).map { section =>
-                allComments ++= comments.findCommentsBySectionId(section.id)
-              }
-              file.id -> allComments.size
-            }.toMap
+        val filteredFiles = filter match {
+          case Some(filt) => files.get(childFiles).found.filter(f => f.filename.toLowerCase.contains(filt.toLowerCase))
+          case None => files.get(childFiles).found
+        }
 
-            val folderHierarchy = new ListBuffer[Folder]()
-            val next = dataset.files.length + dataset.folders.length > limit * (filepageUpdate + 1)
-            Ok(views.html.datasets.filesAndFolders(dataset, None, foldersList, folderHierarchy.reverse.toList, pageIndex, next, limitFileList.toList, fileComments, space)(request.user))
-          }
+        val filteredFolders = filter match {
+          case Some(filt) => folders.get(childFolders).found.filter(f => f.name.toLowerCase.contains(filt.toLowerCase))
+          case None => folders.get(childFolders).found
         }
+
+        val (foldersList: List[Folder], limitFileList: List[File]) =
+          if (play.Play.application().configuration().getBoolean("sortInMemory")) {
+            (SortingUtils.sortFolders(filteredFolders, sortOrder).slice(limit * filepageUpdate, limit * (filepageUpdate + 1)),
+              SortingUtils.sortFiles(filteredFiles, sortOrder).slice(limit * filepageUpdate - filteredFolders.length, limit * (filepageUpdate + 1) - filteredFolders.length))
+          } else {
+            (folders.get(childFolders.reverse.slice(limit * filepageUpdate, limit * (filepageUpdate + 1))).found,
+              files.get(childFiles.reverse.slice(limit * filepageUpdate - childFolders.length, limit * (filepageUpdate + 1) - childFolders.length)).found)
+          }
+
+        // Get comment counts per file
+        val fileComments = limitFileList.map { file =>
+          var allComments = comments.findCommentsByFileId(file.id)
+          sections.findByFileId(file.id).map { section =>
+            allComments ++= comments.findCommentsBySectionId(section.id)
+          }
+          file.id -> allComments.size
+        }.toMap
+
+        // Pagination
+        val next = childFiles.length + childFolders.length > limit * (filepageUpdate + 1)
+
+        Ok(views.html.datasets.filesAndFolders(dataset, folderId, foldersList, folderHierarchy.reverse.toList, pageIndex, next, limitFileList.toList, fileComments, space, filter)(request.user))
       }
       case None => InternalServerError(s"Dataset with id $datasetId not Found")
     }

app/controllers/Files.scala

@@ -220,6 +220,34 @@ class Files @Inject() (
           }
         }
 
+        val pager: models.Pager = dataset match {
+          case None => Pager(None, None)
+          case Some(dsId) => {
+            datasets.get(new UUID(dsId)) match {
+              case None => Pager(None, None)
+              case Some(ds) => {
+                val lastIndex = ds.files.length - 1
+                val index = ds.files.indexOf(id)
+
+                // Set prevFile / nextFile, if applicable
+                if (index > 0 && index < lastIndex) {
+                  // Yields UUID of prevFile and nextFile respectively
+                  Pager(Some(ds.files(index + 1)), Some(ds.files(index - 1)))
+                }else if (index == 0 && index < lastIndex) {
+                  // This is the first file in the list, but not the last
+                  Pager(Some(ds.files(index + 1)), None)
+                } else if (index > 0 && index == lastIndex) {
+                  // This is the last file in the list, but not the first
+                  Pager(None, Some(ds.files(index - 1)))
+                } else {
+                  // There is one item on the list, disable paging
+                  Pager(None, None)
+                }
+              }
+            }
+          }
+        }
+
         //call Polyglot to get all possible output formats for this file's content type
         current.plugin[PolyglotPlugin] match {
           case Some(plugin) => {
@@ -238,7 +266,7 @@ class Files @Inject() (
             plugin.getOutputFormats(contentTypeEnding).map(outputFormats =>
               Ok(views.html.file(file, id.stringify, commentsByFile, previewsWithPreviewer, sectionsWithPreviews,
                 extractorsActive, decodedDatasetsContaining.toList, foldersContainingFile,
-                mds, isRDFExportEnabled, extractionGroups, outputFormats, space, access, folderHierarchy.reverse.toList, decodedSpacesContaining.toList, allDecodedDatasets.toList, view_count, view_date)))
+                mds, isRDFExportEnabled, extractionGroups, outputFormats, space, access, folderHierarchy.reverse.toList, decodedSpacesContaining.toList, allDecodedDatasets.toList, view_count, view_date, pager)))
           }
           case None =>
             Logger.debug("Polyglot plugin not found")
@@ -249,7 +277,7 @@ class Files @Inject() (
             //passing None as the last parameter (list of output formats)
             Future(Ok(views.html.file(file, id.stringify, commentsByFile, previewsWithPreviewer, sectionsWithPreviews,
               extractorsActive, decodedDatasetsContaining.toList, foldersContainingFile,
-              mds, isRDFExportEnabled, extractionGroups, None, space, access, folderHierarchy.reverse.toList, decodedSpacesContaining.toList, allDecodedDatasets.toList, view_count, view_date)))
+              mds, isRDFExportEnabled, extractionGroups, None, space, access, folderHierarchy.reverse.toList, decodedSpacesContaining.toList, allDecodedDatasets.toList, view_count, view_date, pager)))
         }
       }
 

app/models/Pager.scala

@@ -0,0 +1,22 @@
+package models
+
+/**
+ * A simple structure to assist in handling paging through a list of resources.
+ * `prev` and `next` contain the ID of the previous and next resources (respectively).
+ *
+ * Currently this class is used for paging through files in a dataset, but could
+ * be used to handle paging for any list of arbitrary UUIDs / resources in other contexts.
+ *
+ * See:
+ * {@link views.files},
+ * {@link controllers.Files#file}
+ *
+ * @param prev UUID of the previous resource in the list
+ * @param next UUID of the next resource in the list
+ *
+ *     @author lambert8
+ */
+case class Pager(
+  prev: Option[models.UUID],
+  next: Option[models.UUID]
+)

app/models/Relation.scala

@@ -1,5 +1,7 @@
 package models
 
+import java.net.URL
+
 /**
  * Track relations between two arbitrary resources.
  */
@@ -10,10 +12,21 @@ case class Relation (
   rdfType: Option[String] = None // rdfType: Option[URI]
 )
 
+/**
+  * Source or sink node of a relationship.
+  *
+  * @param id a string so that it could be both a local UIUD as well as an external URL
+  * @param resourceType internal resource type (dataset, file, etc.)
+  */
 case class Node (
   id: String,
   resourceType: ResourceType.Value
 )
 
+case class NodeDataset(dataset: Dataset, rdfType: Option[String])
+
+case class NodeFile(file: File, rdfType: Option[String])
+
+case class NodeURL(url: URL, rdfType: Option[String])
 
 

app/services/ElasticsearchPlugin.scala

@@ -441,14 +441,8 @@ class ElasticsearchPlugin(application: Application) extends Plugin {
     // Perform recursion first if necessary
     if (recursive) {
       files.get(dataset.files).found.foreach(f => index(f, idx))
-      for (folderid <- dataset.folders) {
-        folders.get(folderid) match {
-          case Some(f) => {
-            files.get(f.files).found.foreach(fi => index(fi, idx))
-          }
-          case None => Logger.error(s"Error getting file $folderid for recursive indexing")
-        }
-      }
+      for (f <- folders.findByParentDatasetId(dataset.id))
+        files.get(f.files).found.foreach(fi => index(fi, idx))
     }
     index(SearchUtils.getElasticsearchObject(dataset), idx.getOrElse(nameOfIndex))
   }

app/util/SearchUtils.scala

@@ -26,20 +26,16 @@ object SearchUtils {
 
     // Get child_of relationships for File
     var child_of: ListBuffer[String] = ListBuffer()
-    datasets.findByFileIdDirectlyContain(id).map(ds => {
+    // ...first, the dataset which contains the file or its folder
+    datasets.findByFileIdAllContain(id).map(ds => {
       child_of += ds.id.toString
       ds.spaces.map(spid => child_of += spid.toString)
       ds.collections.map(collid => child_of += collid.toString)
     })
-    val folderlist = folders.findByFileId(id).map(fld => {
+    // ...second, the immediate parent folder ID (and the folder's parent) itself
+    folders.findByFileId(id).map(fld => {
       child_of += fld.id.toString
       child_of += fld.parentDatasetId.toString
-      fld.id
-    })
-    datasets.get(folderlist).found.foreach(ds => {
-      child_of += ds.id.toString
-      ds.spaces.map(spid => child_of += spid.toString)
-      ds.collections.map(collid => child_of += collid.toString)
     })
     val child_of_distinct = child_of.toList.distinct
 

app/views/collections/listchildcollection.scala.html

@@ -53,16 +53,6 @@ <h3><a href="@(routes.Collections.collection(collection.id))">@Html(collection.n
                                             <span class='glyphicon glyphicon-star-empty'></span> Unfollow</a>
                                 </div>
                             }
-                            <!-- If user can delete, the button is enabled, otherwise the button is present but disabled to provide consistent UE. -->
-                            @if( user.get.id.equals(collection.author.id) || Permission.checkPermission(Permission.RemoveResourceFromCollection, ResourceRef(ResourceRef.collection, collection.id))){
-                                <button onclick="confirmRemoveResourceFromResource('collection','collection','@(parent.id)','collection','@(collection.id)','@(collection.name)',true,'@(redirect)')"
-                                class="btn btn-link" title="Remove the child collection from the collection @parent.name">
-                                    <span class="glyphicon glyphicon-remove"></span> Remove</button>
-                            } else {
-                                <div class="inline" title="No permission to delete the child collection">
-                                    <button class="btn btn-link btn-sm disabled"><span class="glyphicon glyphicon-remove"></span> Remove</button>
-                                </div>
-                            }
                         }
                     </div>
                 </div>