Merge remote-tracking branch 'origin/develop' into release/1.21.0

Author: robkooper

CHANGELOG.md

@@ -17,6 +17,9 @@ script prior to upgrading to minimize the downtime.
 
 ### Added
 - api.Files jsonfile, adds two fields "downloads" and "views" [#228](https://github.com/clowder-framework/clowder/issues/228)
+- Dataset and file scala.html pages incl schema.org jsonld metadata for (google)datasetsearch [#335](https://github.com/clowder-framework/clowder/issues/335)
+- MiniUser and LicenseData now have to_jsonld methods to return string part of [#335](https://github.com/clowder-framework/clowder/issues/335) metadata
+- LicenseData has urlViaAttributes used by it's to_jsonld to guess url when empty, for [#335](https://github.com/clowder-framework/clowder/issues/335)
 - MRI previewer for NIFTY (.nii) files.
 - Dataset page usually defaults to Files tab, but if no files will now show Metadata first
 - HEIC (.heic) and HEIF (.heif) mimetypes to support new Apple iPhone image file format.
@@ -31,6 +34,9 @@ script prior to upgrading to minimize the downtime.
 - github actions would fail for docker builds due to secrets not existing
 - Fix to remove dataset from a space [#349](https://github.com/clowder-framework/clowder/issues/349)
 
+### Changed
+- Utils.baseURL now on RequestHeader instead of Request[Any]
+
 ## 1.20.3 - 2022-06-10
 
 ### Fixed
@@ -64,6 +70,7 @@ script prior to upgrading to minimize the downtime.
 - Documentation: Added "How to contribute documentation" page
 - Documentation: New Sphinx plugins for dropdowns and menus.
 
+
 ## 1.20.0 - 2022-02-07
 
 ### Added 

app/controllers/Datasets.scala

@@ -836,4 +836,4 @@ class Datasets @Inject() (
     implicit val user = request.user
     Ok(views.html.generalMetadataSearch())
   }
-}
+}

app/controllers/Utils.scala

@@ -13,12 +13,13 @@ object Utils {
   /**
    * Return base url given a request. This will add http or https to the front, for example
    * https://localhost:9443 will be returned if it is using https.
+   *
    */
-  def baseUrl(request: Request[Any], absolute: Boolean = true) = {
+  def baseUrl(request: RequestHeader, absolute: Boolean = true) = {  
     if (absolute) {
-      routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
+       routes.Files.list().absoluteURL(https(request))(request).replace("/files", "")
     } else {
-      routes.Files.list().url.replace("/files", "")
+       routes.Files.list().url.replace("/files", "")
     }
   }
 
@@ -171,4 +172,4 @@ object Utils {
       decodedReplies.toList
     }
   }
-}
+}

app/models/Dataset.scala

@@ -5,6 +5,7 @@ import java.util.Date
 import play.api.libs.json.{Writes, Json}
 import play.api.libs.json._
 import play.api.libs.functional.syntax._
+import _root_.util.Formatters
 
 /**
  * A dataset is a collection of files, and streams.
@@ -38,6 +39,53 @@ case class Dataset(
   def isDefault:Boolean = status == DatasetStatus.DEFAULT.toString
   def isTRIAL:Boolean = status == DatasetStatus.TRIAL.toString
   def inSpace:Boolean = spaces.size > 0
+
+  /**
+    * Caps a list at 'max'  
+    * then turns it's ID's into resolvable URLs of that 'apiRoute' type
+    * end with appending "..." to the List, to signify that it was abridged 
+    *
+    * todo: issue 354 to the max configurable
+    */
+  def cap_api_list (l: List[UUID], max: Int, URLb: String, apiRoute: String) : List[String] = {  
+      if (l.length <= max)  {
+        return l.map(f => URLb + apiRoute + f) 
+      } else {
+         val cl = l.take(max)
+         val r : List[String] = cl.map(f => URLb + apiRoute + f) 
+         return r.::("...").reverse 
+      }
+   } 
+
+  /**
+    * return Dataset as JsValue in jsonld format
+    */
+  def to_jsonld(url: String) : JsValue = { 
+     val so = JsObject(Seq("@vocab" -> JsString("https://schema.org/")))
+     val URLb = url.replaceAll("/$", "") 
+     var pic_id = thumbnail_id.getOrElse("")
+     if (pic_id != "") {
+        pic_id = URLb + pic_id 
+     } else {
+        ""
+     }
+     val datasetLD = Json.obj(
+           "@context" -> so,
+           "identifier" -> id.toString,
+           "name" -> name,
+           "author" -> author.to_jsonld(),
+           "description" -> description,
+           "dateCreated" -> Formatters.iso8601(created), 
+           "DigitalDocument" -> Json.toJson(cap_api_list(files, 10, URLb, "/files/")), 
+           "Collection" -> Json.toJson(cap_api_list(spaces, 10, URLb, "/spaces/")), 
+           "thumbnail" -> Json.toJson(pic_id),
+           "license" -> licenseData.to_jsonld(),
+           "dateModfied" -> Formatters.iso8601(lastModifiedDate),
+           "keywords" -> tags.map(x => x.to_json()),
+           "creator" -> Json.toJson(creators)
+           )
+     return datasetLD
+     }
 }
 
 object DatasetStatus extends Enumeration {
@@ -68,8 +116,9 @@ object Dataset {
 }
 
 
+
 case class DatasetAccess(
   showAccess:  Boolean = false,
   access: String = "N/A",
   accessOptions: List[String] = List.empty
-)
+)

app/models/File.scala

@@ -4,6 +4,8 @@ import java.util.Date
 
 import models.FileStatus.FileStatus
 import play.api.libs.json.{JsObject, Json, Writes}
+import play.api.libs.json._
+import _root_.util.Formatters
 
 /**
  * Uploaded files.
@@ -32,7 +34,33 @@ case class File(
   licenseData: LicenseData = new LicenseData(),
   followers: List[UUID] = List.empty,
   stats: Statistics = new Statistics(),
-  status: String = FileStatus.UNKNOWN.toString) // can't use enums in salat
+  status: String = FileStatus.UNKNOWN.toString) { // can't use enums in salat
+    /**
+    * return File as JsValue in jsonld format
+    */
+     def to_jsonld() : JsValue = {
+     val so = JsObject(Seq("@vocab" -> JsString("https://schema.org/")))
+     val fileLD = Json.obj(
+           "@context" -> so,
+           "identifier" -> id.toString,
+           "name" -> filename,
+           "author" -> author.to_jsonld(),
+           "isBasedOn" -> originalname,
+           "uploadDate" -> Formatters.iso8601(uploadDate),
+           "contentType" -> contentType,
+           "MenuSection" -> sections.map(x => x.to_jsonld()),
+           "keywords" -> tags.map(x => x.to_json()),
+           "thumbnail" -> Json.toJson(thumbnail_id.filterNot(_.isEmpty).getOrElse("")),
+           "description" -> description,
+           "license" -> licenseData.to_jsonld(),
+           "FollowAction" -> Json.toJson(followers),
+           "interactionStatistic" -> stats.to_jsonld, 
+           "status" -> status
+           )
+        return fileLD
+     }
+}
+
 
 // what is the status of the file
 object FileStatus extends Enumeration {

app/models/LicenseData.scala

@@ -2,6 +2,9 @@ package models
 
 import api.Permission
 
+import play.api.libs.json._
+
+
 /**
  * case class to handle specific license information. Currently attached to individual Datasets and Files.  
  */
@@ -41,5 +44,67 @@ case class LicenseData (
     def isRightsOwner(aName: String) = {
         m_rightsHolder == aName
     }
-}
 
+    /**
+     * Utility to return a url even if empty, but enough other attributes available to determine it
+     * this is repurposed from:
+     * function updateData(id, imageBase, sourceObject, authorName)  
+     * in updateLicenseInfo.js line:88
+     */
+    def urlViaAttributes() : String = {
+       if (m_licenseUrl != "") return m_licenseUrl
+       var licenseUrl = m_licenseUrl;
+       if (m_licenseType == "license2") {
+         //No checkboxes selected
+         if (!m_ccAllowCommercial && !m_ccAllowDerivative && !m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by-nc-nd/3.0/";
+         }
+         //Only commercial selected
+         else if (m_ccAllowCommercial && !m_ccAllowDerivative && !m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by-nd/3.0/";
+         }
+         //Only remixing selected
+         else if (!m_ccAllowCommercial && m_ccAllowDerivative && !m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by-nc/3.0/";
+         }
+         //Remixing and Sharealike selected
+         else if (!m_ccAllowCommercial && m_ccAllowDerivative && m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by-nc-sa/3.0/";
+         }
+         //All checkboxes selected
+         else if (m_ccAllowCommercial && m_ccAllowDerivative && m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by-sa/3.0/";
+         }
+         //Commercial and Remixing selected
+         else if (m_ccAllowCommercial && m_ccAllowDerivative && !m_ccRequireShareAlike) {
+            licenseUrl = "http://creativecommons.org/licenses/by/3.0/";
+         }
+         //else { rightsHolder = 'Creative Commons';
+         //   licenseText = 'Specific level info'; }
+      }
+      else if (m_licenseType == "license3") {
+         licenseUrl = "http://creativecommons.org/publicdomain/zero/1.0/";
+      }
+      else {
+         licenseUrl = "https://dbpedia.org/page/All_rights_reserved";
+      }
+      return licenseUrl
+   }
+        
+   /**
+    * Utility function, similar to a json Write, to return string version in json-ld  format
+    * Should also return key
+    */
+    def to_jsonld () : JsValue = {
+       val licURI = this.urlViaAttributes()  //URI = URL except in one case:
+       val licURL = if (licURI != "https://dbpedia.org/page/All_rights_reserved") licURI
+                    else ""
+       val licLD = JsObject(Seq(
+                "@id" -> JsString(licURI),
+                "URL" -> JsString(licURL),
+                "@type" -> JsString("license"),
+                "Text" -> JsString(m_licenseText)  //added this DataType
+                ))
+       return licLD
+    }
+}

app/models/Section.scala

@@ -1,5 +1,7 @@
 package models
 
+import play.api.libs.json._
+
 /**
  * A portion of a file.
  *
@@ -17,12 +19,17 @@ case class Section(
   metadataCount: Long = 0,
   @deprecated("use Metadata","since the use of jsonld") jsonldMetadata : List[Metadata]= List.empty,
   thumbnail_id: Option[String] = None,
-  tags: List[Tag] = List.empty)
+  tags: List[Tag] = List.empty) {
+   def to_jsonld() : JsValue = {
+      return Json.toJson(description)
+   }
+  }
+
 
 case class Rectangle(
   x: Double,
   y: Double,
   w: Double,
   h: Double) {
   override def toString() = f"x: $x%.2f, y: $y%.2f, width: $w%.2f, height: $h%.2f"
-}
+}

app/models/ServerStartTime.scala

@@ -2,6 +2,7 @@ package models
 
 import java.util.Date
 
+
 /**
  * Keeps track of server start time
  * Used in Global Object
@@ -10,4 +11,4 @@ import java.util.Date
 object ServerStartTime {
   var startTime: Date=null
 
-}
+}

app/models/Statistic.scala

@@ -10,7 +10,12 @@ case class Statistics (
   downloads: Int = 0,
   last_viewed: Option[Date] = None,
   last_downloaded: Option[Date] = None
-)
+) {
+   def to_jsonld() : JsValue = {
+      return Json.toJson(views)
+   }
+  }
+
 
 case class StatisticUser (
   user_id: UUID,

app/models/Tag.scala

@@ -2,6 +2,9 @@ package models
 
 import java.util.Date
 
+import play.api.libs.json._
+
+
 /**
  * Add and remove tags
  *
@@ -11,4 +14,8 @@ case class Tag(
   name: String,
   userId: Option[String],
   extractor_id: Option[String],
-  created: Date)
+  created: Date) {
+   def to_json() : JsValue = {
+      return Json.toJson(name)
+   }
+  }