Merge branch 'develop' into bugfix/257-collection-api-create

Author: max-zilla

CHANGELOG.md

@@ -4,13 +4,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## unreleased
+## 1.18.1 - 2021-08-16
+
+This release fixes a critical issue where invalid zip files could result in the files not being uploaded correctly. To check to see if you are affected, please use the following query:
+
+```
+db.uploads.find({"status": "CREATED", "contentType": "application/x-zip-compressed"}, {"author.fullName": 1, "author.email": 1, "filename": 1, "uploadDate": 1, "length": 1})
+```
+
+If any files are returned, you should check to see if these files affected and are missing from clowder.
 
 ### Fixed
+- When zip file is uploaded, it will parse the file to check if it is a valid zip file, this couuld result in files not stored in final storage space [#264](https://github.com/clowder-framework/clowder/issues/264)
 - Updated swagger documentation
 - Return 404 not found when calling file/dataset/space api endpoints with an invalid ID [#251](https://github.com/clowder-framework/clowder/issues/251)
+- Line breaks in welcome message breaks swagger build [#187](https://github.com/clowder-framework/clowder/issues/187)
 - Collections created using api route are now indexed upon creation. [#257](https://github.com/clowder-framework/clowder/issues/257)
 
+### Changed
+- Added more information when writing files to make sure files are written correctly
+- Made cilogon group check debug message instead of error message
+
 ## 1.18.0 - 2021-07-08
 
 ### Added

app/controllers/Application.scala

@@ -16,9 +16,9 @@ import scala.collection.mutable.ListBuffer
  * Main application controller.
  */
 @Singleton
-class Application @Inject() (files: FileService, collections: CollectionService, datasets: DatasetService,
-                             spaces: SpaceService, events: EventService, comments: CommentService,
-                             sections: SectionService, users: UserService, selections: SelectionService) extends SecuredController {
+class Application @Inject()(files: FileService, collections: CollectionService, datasets: DatasetService,
+                            spaces: SpaceService, events: EventService, comments: CommentService,
+                            sections: SectionService, users: UserService, selections: SelectionService) extends SecuredController {
   /**
    * Redirect any url's that have a trailing /
    *
@@ -34,10 +34,10 @@ class Application @Inject() (files: FileService, collections: CollectionService,
     Redirect("http://clowder.ncsa.illinois.edu/swagger/?url=" + swagger)
   }
 
-    /**
-    * Returns the swagger documentation customized for this site.
-    */
-  def swagger = Action  { implicit request =>
+  /**
+   * Returns the swagger documentation customized for this site.
+   */
+  def swagger = Action { implicit request =>
     Play.resource("/public/swagger.yml") match {
       case Some(resource) => {
         val https = Utils.https(request)
@@ -62,7 +62,8 @@ class Application @Inject() (files: FileService, collections: CollectionService,
               skipit = true
               "info:\n" +
                 "  title: " + AppConfiguration.getDisplayName + "\n" +
-                "  description: " + AppConfiguration.getWelcomeMessage + "\n" +
+                //  replace line breaks with space so swagger can build
+                "  description: " + AppConfiguration.getWelcomeMessage.replaceAll("\\n|\\r|\\r\\n", " ") + "\n" +
                 "  version: \"" + sys.props.getOrElse("build.version", default = "0.0.0").toString + "\"\n" +
                 "  termsOfService: " + routes.Application.tos().absoluteURL(https) + "\n" +
                 "  contact: " + "\n" +
@@ -89,29 +90,29 @@ class Application @Inject() (files: FileService, collections: CollectionService,
   def index = UserAction(needActive = false) { implicit request =>
     val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService])
 
-  	implicit val user = request.user
+    implicit val user = request.user
 
     var newsfeedEvents = List.empty[Event]
     if (!play.Play.application().configuration().getBoolean("clowder.disable.events", false)) {
       newsfeedEvents = user.fold(List.empty[Event])(u => events.getEvents(u.followedEntities, Some(20)))
-      newsfeedEvents =  newsfeedEvents ::: events.getRequestEvents(user, Some(20))
+      newsfeedEvents = newsfeedEvents ::: events.getRequestEvents(user, Some(20))
       if (user.isDefined) {
         newsfeedEvents = (newsfeedEvents ::: events.getEventsByUser(user.get, Some(20)))
           .sorted(Ordering.by((_: Event).created).reverse).distinct.take(20)
       }
     }
 
     user match {
-      case Some(clowderUser) if (clowderUser.status==UserStatus.Inactive) => {
+      case Some(clowderUser) if (clowderUser.status == UserStatus.Inactive) => {
         Redirect(routes.Error.notActivated())
       }
-      case Some(clowderUser) if !(clowderUser.status==UserStatus.Inactive) => {
+      case Some(clowderUser) if !(clowderUser.status == UserStatus.Inactive) => {
         newsfeedEvents = newsfeedEvents ::: events.getEventsByUser(clowderUser, Some(20))
-        if( play.Play.application().configuration().getBoolean("showCommentOnHomepage")) newsfeedEvents = newsfeedEvents :::events.getCommentEvent(clowderUser, Some(20))
+        if (play.Play.application().configuration().getBoolean("showCommentOnHomepage")) newsfeedEvents = newsfeedEvents ::: events.getCommentEvent(clowderUser, Some(20))
         newsfeedEvents = newsfeedEvents.sorted(Ordering.by((_: Event).created).reverse).distinct.take(20)
         val datasetsUser = datasets.listUser(12, Some(clowderUser), request.user.fold(false)(_.superAdminMode), clowderUser)
         val collectionList = collections.listUser(12, Some(clowderUser), request.user.fold(false)(_.superAdminMode), clowderUser)
-        val collectionsWithThumbnails = collectionList.map {c =>
+        val collectionsWithThumbnails = collectionList.map { c =>
           if (c.thumbnail_id.isDefined) {
             c
           } else {
@@ -126,7 +127,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         for (aCollection <- collectionsWithThumbnails) {
           decodedCollections += Utils.decodeCollectionElements(aCollection)
         }
-        val spacesUser = spaces.listUser(12, Some(clowderUser),request.user.fold(false)(_.superAdminMode), clowderUser)
+        val spacesUser = spaces.listUser(12, Some(clowderUser), request.user.fold(false)(_.superAdminMode), clowderUser)
         var followers: List[(UUID, String, String, String)] = List.empty
         for (followerID <- clowderUser.followers.take(3)) {
           val userFollower = users.findById(followerID)
@@ -190,12 +191,12 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         }
         Logger.debug("User selections" + user)
         val userSelections: List[String] =
-          if(user.isDefined) selections.get(user.get.identityId.userId).map(_.id.stringify)
+          if (user.isDefined) selections.get(user.get.identityId.userId).map(_.id.stringify)
           else List.empty[String]
         Logger.debug("User selection " + userSelections)
         Ok(views.html.home(AppConfiguration.getDisplayName, newsfeedEvents, clowderUser, datasetsUser,
           decodedCollections.toList, spacesUser, true, followers, followedUsers.take(12), followedFiles.take(8),
-          followedDatasets.take(8), followedCollections.take(8),followedSpaces.take(8), Some(true), userSelections))
+          followedDatasets.take(8), followedCollections.take(8), followedSpaces.take(8), Some(true), userSelections))
       }
       case _ => {
         // Set bytes from appConfig
@@ -234,10 +235,10 @@ class Application @Inject() (files: FileService, collections: CollectionService,
     val sanitezedWelcomeText = sanitizeHTML(AppConfiguration.getWelcomeMessage)
 
     Ok(views.html.index(datasetsCount, filesCount, filesBytes, collectionsCount,
-        spacesCount, usersCount, AppConfiguration.getDisplayName, sanitezedWelcomeText))
+      spacesCount, usersCount, AppConfiguration.getDisplayName, sanitezedWelcomeText))
   }
 
-  def email(subject: String, body: String) = UserAction(needActive=false) { implicit request =>
+  def email(subject: String, body: String) = UserAction(needActive = false) { implicit request =>
     if (request.user.isEmpty) {
       Redirect(routes.Application.index())
     } else {
@@ -252,10 +253,10 @@ class Application @Inject() (files: FileService, collections: CollectionService,
     Ok(views.html.tos(redirect))
   }
 
-  def options(path:String) = UserAction(needActive = false) { implicit request =>
+  def options(path: String) = UserAction(needActive = false) { implicit request =>
     Logger.debug("---controller: PreFlight Information---")
     Ok("")
-   }
+  }
 
   def healthz() = Action { implicit request =>
     Ok("healthy")
@@ -269,7 +270,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
   }
 
   /**
-   *  Javascript routing.
+   * Javascript routing.
    */
   def javascriptRoutes = Action { implicit request =>
     Ok(
@@ -543,7 +544,7 @@ class Application @Inject() (files: FileService, collections: CollectionService,
         controllers.routes.javascript.FileLinks.createLink,
         controllers.routes.javascript.Search.search
       )
-    ).as(JSON) 
+    ).as(JSON)
   }
 
 }

app/fileutils/FilesUtils.java

@@ -71,8 +71,8 @@ else if(!fileName.toLowerCase().endsWith(".ptm")){
             if(allPTMsFlag.equals("found"))
             	return "multi/files-ptm-zipped";
 
-		} catch (IOException e) {
-			// TODO Auto-generated catch block
+		} catch (Throwable e) {
+			Logger.error("Could not read zipfile", e);
 			return ("ERROR: " + e.getMessage());
 		}		
 		return mainFileType;

app/services/CILogonProvider.scala

@@ -57,7 +57,7 @@ class CILogonProvider(application: Application) extends OAuth2Provider(applicati
               }
             }
             case (Some(_), None) => throw new AuthenticationException()
-            case (None, _) => Logger.error("[securesocial] No check needed for groups")
+            case (None, _) => Logger.debug("[securesocial] No check needed for groups")
           }
           user.copy(
             identityId = IdentityId(userId, id),

app/util/FileUtils.scala

@@ -399,14 +399,19 @@ object FileUtils {
     val fileExecutionContext: ExecutionContext = Akka.system().dispatchers.lookup("akka.actor.contexts.file-processing")
     Future {
       try {
-        saveFile(file, f.ref.file, originalZipFile, clowderurl, apiKey, Some(user)).foreach { fixedfile =>
-          processFileBytes(fixedfile, f.ref.file, dataset)
-          files.setStatus(fixedfile.id, FileStatus.UPLOADED)
-          sinkService.logFileUploadEvent(fixedfile, dataset, Option(user))
-          processFile(fixedfile, clowderurl, index, flagsFromPrevious, showPreviews, dataset, runExtractors, apiKey)
-          processDataset(file, dataset, folder, clowderurl, user, index, runExtractors, apiKey)
-          files.setStatus(fixedfile.id, FileStatus.PROCESSED)
+        saveFile(file, f.ref.file, originalZipFile, clowderurl, apiKey, Some(user)) match {
+          case Some(fixedfile) => {
+            processFileBytes(fixedfile, f.ref.file, dataset)
+            files.setStatus(fixedfile.id, FileStatus.UPLOADED)
+            sinkService.logFileUploadEvent(fixedfile, dataset, Option(user))
+            processFile(fixedfile, clowderurl, index, flagsFromPrevious, showPreviews, dataset, runExtractors, apiKey)
+            processDataset(file, dataset, folder, clowderurl, user, index, runExtractors, apiKey)
+            files.setStatus(fixedfile.id, FileStatus.PROCESSED)
+          }
+          case None => Logger.error(s"File was not saved for ${file.id}, saveFile returned None")
         }
+      } catch {
+        case e: Throwable => Logger.error(s"File was not saved for ${file.id}", e)
       } finally {
         f.ref.clean()
       }
@@ -455,12 +460,19 @@ object FileUtils {
     // process rest of file in background
     val fileExecutionContext: ExecutionContext = Akka.system().dispatchers.lookup("akka.actor.contexts.file-processing")
     Future {
-      saveURL(file, url, clowderurl, apiKey, Some(user)).foreach { fixedfile =>
-        processFileBytes(fixedfile, new java.io.File(path), fileds)
-        files.setStatus(fixedfile.id, FileStatus.UPLOADED)
-        processFile(fixedfile, clowderurl, index, flagsFromPrevious, showPreviews, fileds, runExtractors, apiKey)
-        processDataset(file, fileds, folder, clowderurl, user, index, runExtractors, apiKey)
-        files.setStatus(fixedfile.id, FileStatus.PROCESSED)
+      try {
+        saveURL(file, url, clowderurl, apiKey, Some(user)) match{
+          case Some(fixedfile) => {
+            processFileBytes(fixedfile, new java.io.File(path), fileds)
+            files.setStatus(fixedfile.id, FileStatus.UPLOADED)
+            processFile(fixedfile, clowderurl, index, flagsFromPrevious, showPreviews, fileds, runExtractors, apiKey)
+            processDataset(file, fileds, folder, clowderurl, user, index, runExtractors, apiKey)
+            files.setStatus(fixedfile.id, FileStatus.PROCESSED)
+          }
+          case None => Logger.error(s"File was not saved for ${file.id}, saveFile returned None")
+        }
+      } catch {
+        case e: Throwable => Logger.error(s"File was not saved for ${file.id}", e)
       }
     }(fileExecutionContext)
 

doc/src/sphinx/conf.py

@@ -22,7 +22,7 @@
 author = 'Luigi Marini'
 
 # The full version, including alpha/beta/rc tags
-release = '1.18.0'
+release = '1.18.1'
 
 
 # -- General configuration ---------------------------------------------------

project/Build.scala

@@ -13,7 +13,7 @@ import NativePackagerKeys._
 object ApplicationBuild extends Build {
 
   val appName = "clowder"
-  val version = "1.18.0"
+  val version = "1.18.1"
   val jvm = "1.7"
 
   def appVersion: String = {

public/swagger.yml

@@ -9,7 +9,7 @@ info:
     Clowder is a customizable and scalable data management system to support any
     data format and multiple research domains. It is under active development
     and deployed for a variety of research projects.
-  version: 1.18.0
+  version: 1.18.1
   termsOfService: https://clowder.ncsa.illinois.edu/clowder/tos
   contact:
     name: Clowder