Merge pull request #108 from clowder-framework/release/1.12.0

Release/1.12.0

Author: lmarini

.github/workflows/ci.yml

@@ -33,14 +33,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: github branch
         run: |
-          BRANCH=${GITHUB_REF##*/}
-          echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
           if [ "$BRANCH" == "master" ]; then
-            echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
+            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
           elif [ "$BRANCH" == "develop" ]; then
-            echo "::set-env name=CLOWDER_VERSION::develop"
+            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
           else 
-            echo "::set-env name=CLOWDER_VERSION::testing"
+            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
       - uses: actions/setup-java@v1
         with:
@@ -83,14 +87,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: github branch
         run: |
-          BRANCH=${GITHUB_REF##*/}
-          echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
           if [ "$BRANCH" == "master" ]; then
-            echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
+            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
           elif [ "$BRANCH" == "develop" ]; then
-            echo "::set-env name=CLOWDER_VERSION::develop"
+            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
           else 
-            echo "::set-env name=CLOWDER_VERSION::testing"
+            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
       - uses: actions/setup-java@v1
         with:
@@ -123,14 +131,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: github branch
         run: |
-          BRANCH=${GITHUB_REF##*/}
-          echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
           if [ "$BRANCH" == "master" ]; then
-            echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
+            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
           elif [ "$BRANCH" == "develop" ]; then
-            echo "::set-env name=CLOWDER_VERSION::develop"
+            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
           else 
-            echo "::set-env name=CLOWDER_VERSION::testing"
+            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
       - uses: actions/setup-java@v1
         with:
@@ -185,14 +197,18 @@ jobs:
       - uses: actions/checkout@v2
       - name: github branch
         run: |
-          BRANCH=${GITHUB_REF##*/}
-          echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
           if [ "$BRANCH" == "master" ]; then
-            echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
+            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
           elif [ "$BRANCH" == "develop" ]; then
-            echo "::set-env name=CLOWDER_VERSION::develop"
+            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
           else 
-            echo "::set-env name=CLOWDER_VERSION::testing"
+            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
           fi
       - uses: actions/setup-java@v1
         with:

.github/workflows/docker.yml

@@ -65,8 +65,12 @@ jobs:
       # calculate some variables that are used later
       - name: github branch
         run: |
-          BRANCH=${GITHUB_REF##*/}
-          echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
+          if [ "${{ github.event.release.target_commitish }}" != "" ]; then
+            BRANCH="${{ github.event.release.target_commitish }}"
+          else
+            BRANCH=${GITHUB_REF##*/}
+          fi
+          echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
           if [ "$BRANCH" == "master" ]; then
             version="$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
             tags="latest"
@@ -76,14 +80,14 @@ jobs:
               tags="${tags},${version}"
               version=${version%.*}
             done
-            echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
-            echo "::set-env name=CLOWDER_TAGS::${tags}"
+            echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
+            echo "CLOWDER_TAGS=${tags}" >> $GITHUB_ENV
           elif [ "$BRANCH" == "develop" ]; then
-            echo "::set-env name=CLOWDER_VERSION::develop"
-            echo "::set-env name=CLOWDER_TAGS::develop"
+            echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
+            echo "CLOWDER_TAGS=develop" >> $GITHUB_ENV
           else 
-            echo "::set-env name=CLOWDER_VERSION::testing"
-            echo "::set-env name=CLOWDER_TAGS::"
+            echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
+            echo "CLOWDER_TAGS=" >> $GITHUB_ENV
           fi
 
       # build the docker image, this will always run to make sure

CHANGELOG.md

@@ -4,10 +4,38 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.12.0 - 2020-10-19
+**_Warning:_**
+- This update modifies the MongoDB schema. Make sure to start the application with `-DMONGOUPDATE=1`.
+- This update modifies information stored in Elasticsearch used for text based searching. Make sure to initiate a reindex 
+  of Elasticsearch from the Admin menu or by `POST /api/reindex`.
+
+### Added
+- Global extractors page now shows more information, including submission metrics, logs (using Graylog), job history and
+  extractors maturity. Extractors can be grouped using labels. User can filter list of extractors by labels, space, trigger
+  and metadata key.
+- Users have more refined options to set extractors triggers at the space level. They can now follow global settings, 
+  disable and enable triggers.
+- Ability to set chunksize when downloading files. Set defult to 1MB from 8KB. This will result in faster downloads and 
+  less CPU usage at the cost of slightly more memory use.
+- Support for parsing of Date and Numeric data in new metadata fields. New search operators <, >, <=, >= have been 
+  added to search API now that they can be compared properly.
+- Track user_id with every extraction event. [#94](https://github.com/clowder-framework/clowder/issues/94)
+- Added a new storage report at `GET api/reports/storage/spaces/:id` for auditing user storage usage on a space basis.
+- The file and dataset metrics reports also have support for since and until ISO8601 date parameters.
+- Added `viewer_hop` a 3D models previewer for `*.ply` and `*.nxz` files. Added `mimetype.nxz=model/nxz` and 
+  `mimetype.NXZ=model/nxz` as new mimetypes in `conf/mimetypes.conf`
+
+### Fixed
+- Ignore the `update` field when posting to `/api/extractors`. [#89](https://github.com/clowder-framework/clowder/issues/89)
+- Search results were hardcoded to be in batches of 2.
+- Fixed permissions checks on search results for search interfaces that would cause misleading counts. [#60](https://github.com/clowder-framework/clowder/issues/60)
+
 ## 1.11.2 - 2020-10-13
 
 ### Fixed
-- Clowder healthcheck was not correct, resulting in docker-compose never thinking it was healthy. This could also result in traefik not setting up the routes. 
+- Clowder healthcheck was not correct, resulting in docker-compose never thinking it was healthy. This could also result 
+  in traefik not setting up the routes. 
 
 ## 1.11.1 - 2020-09-29
 
@@ -40,6 +68,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Typesafe now only offers https access. [#49](https://github.com/clowder-framework/clowder/issues/49)
 - If uploading files by url > 2147483647 it would fail. [#54](https://github.com/clowder-framework/clowder/issues/54)
 
+
 ## 1.10.1 - 2020-07-16
 
 ### Fixed

CONTRIBUTORS.md

@@ -8,6 +8,7 @@ Following is a list of contributors in alphabetical order:
 - Brock Angelo
 - Chen Wang
 - Chris Navarro
+- Chrysovalantis Constantinou
 - Constantinos Sophocleous
 - Gene Roeder
 - Gregory Jansen
@@ -21,6 +22,7 @@ Following is a list of contributors in alphabetical order:
 - Max Burnette
 - Michal Ondrejcek
 - Michelle Pitcel
+- Mike Bobak
 - Mike Lambert
 - Nicholas Tenczar
 - Nishant Nayudu

README.md

@@ -56,7 +56,7 @@ use this file to setup Let's Encrypt, or tell clowder to use different security
 ## Initializing Clowder
 
 Once clowder has started you will need to create an account. This account can be created using a docker
-container. You can start it with `docker run -ti --rm --network clowder-clowder clowder/mongo-init`. The
+container. You can start it with `docker run -ti --rm --network clowder_clowder clowder/mongo-init`. The
 container will ask for an email address, name, password as well as if this user should be admin (true).
 Once the container finishes running, you can login to clowder. 
 

app/api/Datasets.scala

@@ -55,6 +55,8 @@ class  Datasets @Inject()(
   appConfig: AppConfigurationService,
   esqueue: ElasticsearchQueue) extends ApiController {
 
+  lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
+
   def get(id: UUID) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
     datasets.get(id) match {
       case Some(d) => Ok(toJson(d))
@@ -1833,7 +1835,7 @@ class  Datasets @Inject()(
       case true => {
         current.plugin[RDFExportService].get.getRDFUserMetadataDataset(id.toString, mappingNumber) match{
           case Some(resultFile) =>{
-            Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile)))
+            Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile), chunksize))
               .withHeaders(CONTENT_TYPE -> "application/rdf+xml")
               .withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(resultFile.getName(),request.headers.get("user-agent").getOrElse(""))))
           }
@@ -2749,7 +2751,7 @@ class  Datasets @Inject()(
         // Setup userList, add all users of all spaces associated with the dataset
         dataset.spaces.foreach { spaceId =>
           spaces.get(spaceId) match {
-            case Some(spc) => userList = spaces.getUsersInSpace(spaceId) ::: userList
+            case Some(spc) => userList = spaces.getUsersInSpace(spaceId, None) ::: userList
             case None => NotFound(s"Error: No $spaceTitle found for $id.")
           }
         }

app/api/Extractions.scala

@@ -3,8 +3,8 @@ package api
 import java.io.{FileInputStream, InputStream}
 import java.net.URL
 import java.util.Calendar
-import javax.inject.Inject
 
+import javax.inject.Inject
 import controllers.Utils
 import fileutils.FilesUtils
 import models._
@@ -475,7 +475,32 @@ class Extractions @Inject()(
       },
       info => {
         extractors.updateExtractorInfo(info) match {
-          case Some(u) => Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+          case Some(u) => {
+            // Create/assign any default labels for this extractor
+            u.defaultLabels.foreach(labelStr => {
+              val segments = labelStr.split("/")
+              val (labelName, labelCategory) = if (segments.length > 1) {
+                (segments(1), segments(0))
+              } else {
+                (segments(0), "Other")
+              }
+              extractors.getExtractorsLabel(labelName) match {
+                case None => {
+                  // Label does not exist - create and assign it
+                  val createdLabel = extractors.createExtractorsLabel(labelName, Some(labelCategory), List[String](u.name))
+                }
+                case Some(lbl) => {
+                  // Label already exists, assign it
+                  if (!lbl.extractors.contains(u.name)) {
+                    val label = ExtractorsLabel(lbl.id, lbl.name, lbl.category, lbl.extractors ++ List[String](u.name))
+                    val updatedLabel = extractors.updateExtractorsLabel(label)
+                  }
+                }
+              }
+            })
+
+            Ok(Json.obj("status" -> "OK", "message" -> ("Extractor info updated. ID = " + u.id)))
+          }
           case None => BadRequest(Json.obj("status" -> "KO", "message" -> "Error updating extractor info"))
         }
       }
@@ -674,4 +699,71 @@ class Extractions @Inject()(
     Ok(toJson("added new event"))
   }
 
+  def createExtractorsLabel() = ServerAdminAction(parse.json) { implicit request =>
+    // Fetch parameters from request body
+    val (name, category, assignedExtractors) = parseExtractorsLabel(request)
+
+    // Validate that name is not empty
+    if (name.isEmpty) {
+      BadRequest("Label Name cannot be empty")
+    } else {
+      // Validate that name is unique
+      extractors.getExtractorsLabel(name) match {
+        case Some(lbl) => Conflict("Label name is already in use: " + lbl.name)
+        case None => {
+          // Create the new label
+          val label = extractors.createExtractorsLabel(name, category, assignedExtractors)
+          Ok(Json.toJson(label))
+        }
+      }
+    }
+  }
+
+  def updateExtractorsLabel(id: UUID) = ServerAdminAction(parse.json) { implicit request =>
+    // Fetch parameters from request body
+    val (name, category, assignedExtractors) = parseExtractorsLabel(request)
+
+    // Validate that name is not empty
+    if (name.isEmpty) {
+      BadRequest("Label Name cannot be empty")
+    } else {
+      // Validate that name is still unique
+      extractors.getExtractorsLabel(name) match {
+        case Some(lbl) => {
+          // Exclude current id (in case name hasn't changed)
+          if (lbl.id != id) {
+            Conflict("Label name is already in use: " + lbl.name)
+          } else {
+            // Update the label
+            val updatedLabel = extractors.updateExtractorsLabel(ExtractorsLabel(id, name, category, assignedExtractors))
+            Ok(Json.toJson(updatedLabel))
+          }
+        }
+        case None => {
+          // Update the label
+          val updatedLabel = extractors.updateExtractorsLabel(ExtractorsLabel(id, name, category, assignedExtractors))
+          Ok(Json.toJson(updatedLabel))
+        }
+      }
+    }
+  }
+
+  def deleteExtractorsLabel(id: UUID) = ServerAdminAction { implicit request =>
+    // Fetch existing label
+    extractors.getExtractorsLabel(id) match {
+      case Some(lbl) => {
+        val deleted = extractors.deleteExtractorsLabel(lbl)
+        Ok(Json.toJson(deleted))
+      }
+      case None => BadRequest("Failed to delete label: " + id)
+    }
+  }
+
+  def parseExtractorsLabel(request: UserRequest[JsValue]): (String, Option[String], List[String]) = {
+    val name = (request.body \ "name").as[String]
+    val category = (request.body \ "category").asOpt[String]
+    val assignedExtractors = (request.body \ "extractors").as[List[String]]
+
+    (name, category, assignedExtractors)
+  }
 }