Skip to content

Commit a9c2700

Browse files
robkooperrobkooper
authored
Merge pull request #323 from clowder-framework/release/1.20.0
Release/1.20.0
2 parents 17651a4 + 951e16a commit a9c2700

20 files changed

+211
-99
lines changed

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## 1.20.0 - 2022-02-07
8+
9+
### Fixed
10+
- Conf file and code had incosistent spelling of BagIt. Now all have capital B and I.
11+
- When event stream is disabled don't show for logged in user [#280](https://github.com/clowder-framework/clowder/issues/280)
12+
- three.js is no longer associated with application/octet-stream, now with models [#305](https://github.com/clowder-framework/clowder/issues/305)
13+
14+
### Changed
15+
- Download of dataset/collection now has optional parameter bagit (default false) to download items in bagit format.
16+
717
## 1.19.5 - 2022-01-21
818

919
### Fixed

‎app/Iterators/CollectionIterator.scala‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,12 @@ import scala.collection.mutable.ListBuffer
1313

1414
//this is to download collections
1515
//that are not at the root level
16-
class CollectionIterator(pathToFolder : String, parent_collection : models.Collection,zip : ZipOutputStream, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
17-
collections: CollectionService, datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
16+
class CollectionIterator(pathToFolder: String, parent_collection: models.Collection, zip: ZipOutputStream,
17+
md5Files: scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
18+
bagit: Boolean, collections: CollectionService, datasets: DatasetService, files: FileService,
19+
folders: FolderService, metadataService: MetadataService,
1820
spaces : SpaceService) extends Iterator[Option[InputStream]] {
1921

20-
2122
def getNextGenerationCollections(currentCollections : List[Collection]) : List[Collection] = {
2223
var nextGenerationCollections : ListBuffer[Collection] = ListBuffer.empty[Collection]
2324
for (currentCollection <- currentCollections){
@@ -28,7 +29,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
2829
nextGenerationCollections.toList
2930
}
3031

31-
val datasetIterator = new DatasetsInCollectionIterator(pathToFolder,parent_collection,zip,md5Files,user,
32+
val datasetIterator = new DatasetsInCollectionIterator(pathToFolder,parent_collection,zip,bagit, md5Files,user,
3233
datasets,files, folders, metadataService,spaces)
3334

3435
var currentCollectionIterator : Option[CollectionIterator] = None
@@ -40,7 +41,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
4041
var childCollectionCount = 0
4142
var numChildCollections = child_collections.size
4243

43-
var file_type = 0
44+
var file_type = if (bagit) 0 else 2
4445

4546
// TODO: Repeat from api/Collections
4647
def jsonCollection(collection: Collection): JsValue = {
@@ -74,12 +75,12 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
7475
if (file_type < 2){
7576
true
7677
}
77-
else if (file_type ==2){
78+
else if (file_type == 2){
7879
if (datasetIterator.hasNext()){
7980
true
8081
} else if (numChildCollections > 0){
81-
82-
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name, child_collections(childCollectionCount),zip,md5Files,user,collections,datasets,files,
82+
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name,
83+
child_collections(childCollectionCount),zip,md5Files,user, bagit, collections,datasets,files,
8384
folders,metadataService,spaces))
8485
file_type +=1
8586
true
@@ -94,7 +95,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
9495
} else if (childCollectionCount < numChildCollections -2){
9596
childCollectionCount+=1
9697
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name, child_collections(childCollectionCount),zip,md5Files,user,
97-
collections,datasets,files,
98+
bagit, collections,datasets,files,
9899
folders,metadataService,spaces))
99100
true
100101
} else {

‎app/Iterators/DatasetIterator.scala‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import scala.collection.mutable.ListBuffer
1414

1515
//this iterator is used for downloading a dataset
1616
//it has a file iterator
17-
class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest],
17+
class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipOutputStream, bagit: Boolean, md5Files :scala.collection.mutable.HashMap[String, MessageDigest],
1818
folders : FolderService, files: FileService, metadataService : MetadataService, datasets: DatasetService, spaces : SpaceService) extends Iterator[Option[InputStream]] {
1919

2020
//get files in the dataset
@@ -111,10 +111,14 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
111111

112112
var fileCounter = 0
113113

114-
var currentFileIterator : Option[FileIterator] = None
114+
var currentFileIterator : Option[FileIterator] = if (numFiles > 0){
115+
Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),bagit, zip,md5Files,files,folders,metadataService))
116+
} else {
117+
None
118+
}
115119

116120
var is : Option[InputStream] = None
117-
var file_type : Int = 0
121+
var file_type : Int = if (bagit) 0 else 2
118122

119123
def hasNext() = {
120124
if (file_type < 2){
@@ -126,7 +130,7 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
126130
true
127131
} else if (fileCounter < numFiles -1){
128132
fileCounter +=1
129-
currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),zip,md5Files,files,folders,metadataService))
133+
currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),bagit, zip,md5Files,files,folders,metadataService))
130134
true
131135
} else {
132136
false
@@ -155,7 +159,6 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
155159
md5Files.put("_metadata.json",md5)
156160
if (numFiles > 0){
157161
file_type+=1
158-
currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),zip,md5Files,files,folders,metadataService))
159162
} else {
160163
file_type+=2
161164
}

‎app/Iterators/DatasetsInCollectionIterator.scala‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import scala.collection.mutable.ListBuffer
1111

1212
//this is used to download the datasets in a collection
1313
//it creates an iterator for each dataset in the collection
14-
class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Collection, zip : ZipOutputStream, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
14+
class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Collection, zip : ZipOutputStream, bagit: Boolean, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
1515
datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
1616
spaces : SpaceService) extends Iterator[Option[InputStream]] {
1717

@@ -36,8 +36,7 @@ class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Co
3636
}
3737

3838
var currentDatasetIterator : Option[DatasetIterator] = if (numDatasets > 0){
39-
40-
Some(new DatasetIterator(pathToFolder+"/"+currentDataset.get.name,currentDataset.get, zip, md5Files,
39+
Some(new DatasetIterator(pathToFolder+"/"+currentDataset.get.name,currentDataset.get, zip, bagit, md5Files,
4140
folders, files,metadataService,datasets,spaces))
4241
} else {
4342
None
@@ -56,7 +55,7 @@ class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Co
5655
currentDataset = Some(datasetsInCollection(datasetCount))
5756
currentDataset match {
5857
case Some(cd) => {
59-
currentDatasetIterator = Some(new DatasetIterator(pathToFolder+"/"+cd.name,cd, zip, md5Files,
58+
currentDatasetIterator = Some(new DatasetIterator(pathToFolder+"/"+cd.name,cd, zip, bagit, md5Files,
6059
folders, files,metadataService,datasets,spaces))
6160
true
6261
}

‎app/Iterators/FileIterator.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import util.JSONLD
1111

1212
//this is used for file downloads
1313
//called by the dataset interator
14-
class FileIterator (pathToFile : String, file : models.File,zip : ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest], files : FileService, folders : FolderService , metadataService : MetadataService) extends Iterator[Option[InputStream]] {
14+
class FileIterator (pathToFile : String, file : models.File, bagit: Boolean, zip : ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest], files : FileService, folders : FolderService , metadataService : MetadataService) extends Iterator[Option[InputStream]] {
1515

1616
def getFileInfoAsJson(file : models.File) : JsValue = {
1717
val rightsHolder = {
@@ -54,7 +54,7 @@ class FileIterator (pathToFile : String, file : models.File,zip : ZipOutputStrea
5454
Some(new ByteArrayInputStream(s.getBytes("UTF-8")))
5555
}
5656

57-
var file_type : Int = 0
57+
var file_type : Int = if (bagit) 0 else 2
5858
var is : Option[InputStream] = None
5959
def hasNext() = {
6060
if ( file_type < 3){

‎app/Iterators/RootCollectionIterator.scala‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ import scala.collection.mutable.ListBuffer
1717
class RootCollectionIterator(pathToFolder : String, root_collection : models.Collection,zip : ZipOutputStream,
1818
md5Files : scala.collection.mutable.HashMap[String, MessageDigest],
1919
md5Bag : scala.collection.mutable.HashMap[String, MessageDigest],
20-
user : Option[User],totalBytes : Long,bagit : Boolean,
20+
user : Option[User],totalBytes : Long, bagit : Boolean,
2121
collections: CollectionService, datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
2222
spaces : SpaceService) extends Iterator[Option[InputStream]] {
2323

24-
val datasetIterator = new DatasetsInCollectionIterator(root_collection.name,root_collection,zip,md5Files,user,
24+
val datasetIterator = new DatasetsInCollectionIterator(root_collection.name,root_collection,zip,bagit, md5Files,user,
2525
datasets,files,folders,metadataService,spaces)
2626

2727
var currentCollectionIterator : Option[CollectionIterator] = None
@@ -34,7 +34,7 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
3434
var numCollections = child_collections.size
3535

3636
var bytesSoFar : Long = 0L
37-
var file_type = 0
37+
var file_type = if (bagit) 0 else 2
3838

3939

4040
private def addCollectionInfoToZip(folderName: String, collection: models.Collection, zip: ZipOutputStream): Option[InputStream] = {
@@ -107,9 +107,9 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
107107
true
108108
} else if (numCollections > 0){
109109

110-
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name, child_collections(collectionCount),zip,md5Files,user,
111-
collections,datasets,files,
112-
folders,metadataService,spaces))
110+
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name,
111+
child_collections(collectionCount), zip, md5Files, user, bagit,
112+
collections, datasets,files, folders, metadataService, spaces))
113113
file_type +=1
114114
true
115115
} else if (bagit){
@@ -127,7 +127,7 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
127127
} else if (collectionCount < numCollections -2){
128128
collectionCount+=1
129129
currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name, child_collections(collectionCount),zip,md5Files,user,
130-
collections,datasets,files,
130+
bagit, collections,datasets,files,
131131
folders,metadataService,spaces))
132132
true
133133
} else {

‎app/Iterators/SelectedIterator.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class SelectedIterator(pathToFolder : String, selected : List[Dataset], zip : Zi
2121

2222
var datasetCount = 0
2323
var currDs = selected(datasetCount)
24-
var datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name, currDs, zip, md5Files, folders, files,
24+
var datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name, currDs, zip, bagit, md5Files, folders, files,
2525
metadataService,datasets,spaces)
2626
var file_type = 0
2727

@@ -54,7 +54,7 @@ class SelectedIterator(pathToFolder : String, selected : List[Dataset], zip : Zi
5454
} else if (selected.length > datasetCount+1){
5555
datasetCount += 1
5656
currDs = selected(datasetCount)
57-
datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name,currDs,zip,md5Files,folders,files,
57+
datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name,currDs,zip,bagit, md5Files,folders,files,
5858
metadataService,datasets,spaces)
5959
true
6060
} else if (bagit) {

‎app/api/Collections.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -801,11 +801,10 @@ class Collections @Inject() (datasets: DatasetService,
801801
}
802802
}
803803

804-
def download(id: UUID, compression: Int) = PermissionAction(Permission.DownloadFiles, Some(ResourceRef(ResourceRef.collection, id))) { implicit request =>
804+
def download(id: UUID, compression: Int, bagit: Boolean) = PermissionAction(Permission.DownloadFiles, Some(ResourceRef(ResourceRef.collection, id))) { implicit request =>
805805
implicit val user = request.user
806806
collections.get(id) match {
807807
case Some(collection) => {
808-
val bagit = play.api.Play.configuration.getBoolean("downloadCollectionBagit").getOrElse(true)
809808
// Use custom enumerator to create the zip file on the fly
810809
// Use a 1MB in memory byte array
811810
Ok.chunked(enumeratorFromCollection(collection,1024*1024, compression,bagit,user)).withHeaders(
@@ -821,7 +820,9 @@ class Collections @Inject() (datasets: DatasetService,
821820
}
822821

823822

824-
def enumeratorFromCollection(collection: Collection, chunkSize: Int = 1024 * 8, compression: Int = Deflater.DEFAULT_COMPRESSION, bagit: Boolean, user : Option[User])
823+
def enumeratorFromCollection(collection: Collection, chunkSize: Int = 1024 * 8,
824+
compression: Int = Deflater.DEFAULT_COMPRESSION,
825+
bagit: Boolean, user : Option[User])
825826
(implicit ec: ExecutionContext): Enumerator[Array[Byte]] = {
826827

827828
implicit val pec = ec.prepare()
@@ -839,8 +840,6 @@ class Collections @Inject() (datasets: DatasetService,
839840
var current_iterator = new RootCollectionIterator(collection.name,collection,zip,md5Files,md5Bag,user, totalBytes,bagit,collections,
840841
datasets,files,folders,metadataService,spaces)
841842

842-
843-
844843
//var current_iterator = new FileIterator(folderNameMap(inputFiles(1).id),inputFiles(1), zip,md5Files)
845844
var is = current_iterator.next()
846845

0 commit comments

Comments
 (0)