Skip to content

Commit cfc22f0

Browse files
bodom0015bodom0015
committed
Merge branch 'develop' of https://github.com/clowder-framework/clowder into extractor-catalog-labels
2 parents 1e0a4bf + c14557f commit cfc22f0

30 files changed

+289
-133
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 36 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,18 @@ jobs:
3333
- uses: actions/checkout@v2
3434
- name: github branch
3535
run: |
36-
BRANCH=${GITHUB_REF##*/}
37-
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
36+
if [ "${{ github.event.release.target_commitish }}" != "" ]; then
37+
BRANCH="${{ github.event.release.target_commitish }}"
38+
else
39+
BRANCH=${GITHUB_REF##*/}
40+
fi
41+
echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
3842
if [ "$BRANCH" == "master" ]; then
39-
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
43+
echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
4044
elif [ "$BRANCH" == "develop" ]; then
41-
echo "::set-env name=CLOWDER_VERSION::develop"
45+
echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
4246
else
43-
echo "::set-env name=CLOWDER_VERSION::testing"
47+
echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
4448
fi
4549
- uses: actions/setup-java@v1
4650
with:
@@ -83,14 +87,18 @@ jobs:
8387
- uses: actions/checkout@v2
8488
- name: github branch
8589
run: |
86-
BRANCH=${GITHUB_REF##*/}
87-
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
90+
if [ "${{ github.event.release.target_commitish }}" != "" ]; then
91+
BRANCH="${{ github.event.release.target_commitish }}"
92+
else
93+
BRANCH=${GITHUB_REF##*/}
94+
fi
95+
echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
8896
if [ "$BRANCH" == "master" ]; then
89-
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
97+
echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
9098
elif [ "$BRANCH" == "develop" ]; then
91-
echo "::set-env name=CLOWDER_VERSION::develop"
99+
echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
92100
else
93-
echo "::set-env name=CLOWDER_VERSION::testing"
101+
echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
94102
fi
95103
- uses: actions/setup-java@v1
96104
with:
@@ -123,14 +131,18 @@ jobs:
123131
- uses: actions/checkout@v2
124132
- name: github branch
125133
run: |
126-
BRANCH=${GITHUB_REF##*/}
127-
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
134+
if [ "${{ github.event.release.target_commitish }}" != "" ]; then
135+
BRANCH="${{ github.event.release.target_commitish }}"
136+
else
137+
BRANCH=${GITHUB_REF##*/}
138+
fi
139+
echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
128140
if [ "$BRANCH" == "master" ]; then
129-
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
141+
echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
130142
elif [ "$BRANCH" == "develop" ]; then
131-
echo "::set-env name=CLOWDER_VERSION::develop"
143+
echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
132144
else
133-
echo "::set-env name=CLOWDER_VERSION::testing"
145+
echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
134146
fi
135147
- uses: actions/setup-java@v1
136148
with:
@@ -185,14 +197,18 @@ jobs:
185197
- uses: actions/checkout@v2
186198
- name: github branch
187199
run: |
188-
BRANCH=${GITHUB_REF##*/}
189-
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
200+
if [ "${{ github.event.release.target_commitish }}" != "" ]; then
201+
BRANCH="${{ github.event.release.target_commitish }}"
202+
else
203+
BRANCH=${GITHUB_REF##*/}
204+
fi
205+
echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
190206
if [ "$BRANCH" == "master" ]; then
191-
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
207+
echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
192208
elif [ "$BRANCH" == "develop" ]; then
193-
echo "::set-env name=CLOWDER_VERSION::develop"
209+
echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
194210
else
195-
echo "::set-env name=CLOWDER_VERSION::testing"
211+
echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
196212
fi
197213
- uses: actions/setup-java@v1
198214
with:

‎.github/workflows/docker.yml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,12 @@ jobs:
6565
# calculate some variables that are used later
6666
- name: github branch
6767
run: |
68-
BRANCH=${GITHUB_REF##*/}
69-
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
68+
if [ "${{ github.event.release.target_commitish }}" != "" ]; then
69+
BRANCH="${{ github.event.release.target_commitish }}"
70+
else
71+
BRANCH=${GITHUB_REF##*/}
72+
fi
73+
echo "GITHUB_BRANCH=${BRANCH}" >> $GITHUB_ENV
7074
if [ "$BRANCH" == "master" ]; then
7175
version="$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
7276
tags="latest"
@@ -76,14 +80,14 @@ jobs:
7680
tags="${tags},${version}"
7781
version=${version%.*}
7882
done
79-
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
80-
echo "::set-env name=CLOWDER_TAGS::${tags}"
83+
echo "CLOWDER_VERSION=$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')" >> $GITHUB_ENV
84+
echo "CLOWDER_TAGS=${tags}" >> $GITHUB_ENV
8185
elif [ "$BRANCH" == "develop" ]; then
82-
echo "::set-env name=CLOWDER_VERSION::develop"
83-
echo "::set-env name=CLOWDER_TAGS::develop"
86+
echo "CLOWDER_VERSION=develop" >> $GITHUB_ENV
87+
echo "CLOWDER_TAGS=develop" >> $GITHUB_ENV
8488
else
85-
echo "::set-env name=CLOWDER_VERSION::testing"
86-
echo "::set-env name=CLOWDER_TAGS::"
89+
echo "CLOWDER_VERSION=testing" >> $GITHUB_ENV
90+
echo "CLOWDER_TAGS=" >> $GITHUB_ENV
8791
fi
8892
8993
# build the docker image, this will always run to make sure

‎CHANGELOG.md‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,31 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7-
8-
## Unreleased
7+
## 1.12.0 - 2020-10-19
8+
**_Warning:_**
9+
- This update modifies the MongoDB schema. Make sure to start the application with `-DMONGOUPDATE=1`.
10+
- This update modifies information stored in Elasticsearch used for text based searching. Make sure to initiate a reindex
11+
of Elasticsearch from the Admin menu or by `POST /api/reindex`.
912

1013
### Added
11-
- Add endpoint and view page for the extractor metrics.
12-
- Add endpoint and view page to view a extractor's log from graylog.
14+
- Ability to set chunksize for clowder when downloading files. This is changed to 1MB from 8KB. This will result in
15+
faster downloads and less CPU usage at the cost of slightly more memory use.
16+
- Support for parsing of Date and Numeric data in new metadata fields. New search operators <, >, <=, >= have been
17+
added to search API now that those data are properly compared.
18+
- Global extractors page now shows more information, including submission metrics, logs (using Graylog), job history and
19+
extractors maturity.
20+
- Users have more refined options to set extractors triggers at the space level. They can now follow global settings,
21+
disable and enable triggers.
22+
23+
### Fixed
1324
- Ignore the `update` field when posting to `/api/extractors`. [#89](https://github.com/clowder-framework/clowder/issues/89)
25+
- Search results were hardcoded to be in batches of 2.
1426

15-
## 1.11.2 - 2020-10-13
27+
# 1.11.2 - 2020-10-13
1628

1729
### Fixed
1830
- Clowder healthcheck was not correct, resulting in docker-compose never thinking it was healthy. This could also result
19-
in traefik not setting up the routes.
31+
in traefik not setting up the routes.
2032

2133
## 1.11.1 - 2020-09-29
2234

@@ -49,6 +61,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
4961
- Typesafe now only offers https access. [#49](https://github.com/clowder-framework/clowder/issues/49)
5062
- If uploading files by url > 2147483647 it would fail. [#54](https://github.com/clowder-framework/clowder/issues/54)
5163

64+
5265
## 1.10.1 - 2020-07-16
5366

5467
### Fixed

‎CONTRIBUTORS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ Following is a list of contributors in alphabetical order:
2121
- Max Burnette
2222
- Michal Ondrejcek
2323
- Michelle Pitcel
24+
- Mike Bobak
2425
- Mike Lambert
2526
- Nicholas Tenczar
2627
- Nishant Nayudu

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ use this file to setup Let's Encrypt, or tell clowder to use different security
5656
## Initializing Clowder
5757

5858
Once clowder has started you will need to create an account. This account can be created using a docker
59-
container. You can start it with `docker run -ti --rm --network clowder-clowder clowder/mongo-init`. The
59+
container. You can start it with `docker run -ti --rm --network clowder_clowder clowder/mongo-init`. The
6060
container will ask for an email address, name, password as well as if this user should be admin (true).
6161
Once the container finishes running, you can login to clowder.
6262

‎app/api/Datasets.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class Datasets @Inject()(
5555
appConfig: AppConfigurationService,
5656
esqueue: ElasticsearchQueue) extends ApiController {
5757

58+
lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
59+
5860
def get(id: UUID) = PermissionAction(Permission.ViewDataset, Some(ResourceRef(ResourceRef.dataset, id))) { implicit request =>
5961
datasets.get(id) match {
6062
case Some(d) => Ok(toJson(d))
@@ -1833,7 +1835,7 @@ class Datasets @Inject()(
18331835
case true => {
18341836
current.plugin[RDFExportService].get.getRDFUserMetadataDataset(id.toString, mappingNumber) match{
18351837
case Some(resultFile) =>{
1836-
Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile)))
1838+
Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile), chunksize))
18371839
.withHeaders(CONTENT_TYPE -> "application/rdf+xml")
18381840
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(resultFile.getName(),request.headers.get("user-agent").getOrElse(""))))
18391841
}

‎app/api/Files.scala‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ class Files @Inject()(
5454
appConfig: AppConfigurationService,
5555
esqueue: ElasticsearchQueue) extends ApiController {
5656

57+
lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
58+
5759
def get(id: UUID) = PermissionAction(Permission.ViewFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request =>
5860
Logger.debug("GET file with id " + id)
5961
files.get(id) match {
@@ -128,14 +130,14 @@ class Files @Inject()(
128130
CONTENT_TYPE -> contentType
129131
)
130132
),
131-
body = Enumerator.fromStream(inputStream)
133+
body = Enumerator.fromStream(inputStream, chunksize)
132134
)
133135
}
134136
}
135137
case None => {
136138
val userAgent = request.headers.get("user-agent").getOrElse("")
137139

138-
Ok.chunked(Enumerator.fromStream(inputStream))
140+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
139141
.withHeaders(CONTENT_TYPE -> contentType)
140142
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, userAgent)))
141143
}
@@ -189,12 +191,12 @@ class Files @Inject()(
189191
CONTENT_TYPE -> contentType
190192
)
191193
),
192-
body = Enumerator.fromStream(inputStream)
194+
body = Enumerator.fromStream(inputStream, chunksize)
193195
)
194196
}
195197
}
196198
case None => {
197-
Ok.chunked(Enumerator.fromStream(inputStream))
199+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
198200
.withHeaders(CONTENT_TYPE -> contentType)
199201
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))
200202
}
@@ -669,7 +671,7 @@ class Files @Inject()(
669671
case true => {
670672
current.plugin[RDFExportService].get.getRDFUserMetadataFile(id.stringify, mappingNumber) match {
671673
case Some(resultFile) => {
672-
Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile)))
674+
Ok.chunked(Enumerator.fromStream(new FileInputStream(resultFile), chunksize))
673675
.withHeaders(CONTENT_TYPE -> "application/rdf+xml")
674676
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(resultFile.getName(), request.headers.get("user-agent").getOrElse(""))))
675677
}
@@ -936,13 +938,13 @@ class Files @Inject()(
936938
CONTENT_TYPE -> contentType
937939
)
938940
),
939-
body = Enumerator.fromStream(inputStream)
941+
body = Enumerator.fromStream(inputStream, chunksize)
940942
)
941943
}
942944
}
943945
case None => {
944946
//IMPORTANT: Setting CONTENT_LENGTH header here introduces bug!
945-
Ok.chunked(Enumerator.fromStream(inputStream))
947+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
946948
.withHeaders(CONTENT_TYPE -> contentType)
947949
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))
948950

@@ -989,13 +991,13 @@ class Files @Inject()(
989991
CONTENT_TYPE -> contentType
990992
)
991993
),
992-
body = Enumerator.fromStream(inputStream)
994+
body = Enumerator.fromStream(inputStream, chunksize)
993995
)
994996
}
995997
}
996998
case None => {
997999
//IMPORTANT: Setting CONTENT_LENGTH header here introduces bug!
998-
Ok.chunked(Enumerator.fromStream(inputStream))
1000+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
9991001
.withHeaders(CONTENT_TYPE -> contentType)
10001002
//.withHeaders(CONTENT_LENGTH -> contentLength.toString)
10011003
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))

‎app/api/Logos.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import scala.concurrent.Future
1515

1616
class Logos @Inject()(logos: LogoService) extends ApiController {
1717

18+
lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
19+
1820
def list(path: Option[String], name: Option[String]) = AuthenticatedAction { implicit request =>
1921
Ok(toJson(logos.list(path, name)))
2022
}
@@ -102,7 +104,7 @@ class Logos @Inject()(logos: LogoService) extends ApiController {
102104
def downloadId(id: UUID, file: Option[String]) = Action.async { implicit request =>
103105
logos.getBytes(id) match {
104106
case Some((inputStream, filename, contentType, contentLength)) =>
105-
Future(Ok.chunked(Enumerator.fromStream(inputStream))
107+
Future(Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
106108
.withHeaders(CONTENT_TYPE -> contentType))
107109
case None => {
108110
file match {

‎app/api/Previews.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ import util.FileUtils
2828
@Singleton
2929
class Previews @Inject()(previews: PreviewService, tiles: TileService) extends ApiController {
3030

31+
lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
32+
3133
def list = PermissionAction(Permission.ViewFile) {
3234
request =>
3335
val list = for (p <- previews.listPreviews()) yield jsonPreview(p)
@@ -79,12 +81,12 @@ class Previews @Inject()(previews: PreviewService, tiles: TileService) extends A
7981
CONTENT_TYPE -> contentType
8082
)
8183
),
82-
body = Enumerator.fromStream(inputStream)
84+
body = Enumerator.fromStream(inputStream, chunksize)
8385
)
8486
}
8587
}
8688
case None => {
87-
Ok.chunked(Enumerator.fromStream(inputStream))
89+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
8890
.withHeaders(CONTENT_TYPE -> contentType)
8991
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(filename, request.headers.get("user-agent").getOrElse(""))))
9092

@@ -238,12 +240,12 @@ class Previews @Inject()(previews: PreviewService, tiles: TileService) extends A
238240
CONTENT_TYPE -> contentType
239241
)
240242
),
241-
body = Enumerator.fromStream(inputStream)
243+
body = Enumerator.fromStream(inputStream, chunksize)
242244
)
243245
}
244246
}
245247
case None => {
246-
Ok.chunked(Enumerator.fromStream(inputStream))
248+
Ok.chunked(Enumerator.fromStream(inputStream, chunksize))
247249
.withHeaders(CONTENT_TYPE -> contentType)
248250
.withHeaders(CONTENT_DISPOSITION -> (FileUtils.encodeAttachment(tilename, request.headers.get("user-agent").getOrElse(""))))
249251

‎app/api/Proxy.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ object Proxy {
4040
class Proxy @Inject()() extends ApiController {
4141
import Proxy.ConfigPrefix
4242

43+
lazy val chunksize = play.Play.application().configuration().getInt("clowder.chunksize", 1024*1024)
44+
4345
/**
4446
* Translates an endpoint_key to a target endpoint based on Clowder's configuration
4547
*/
@@ -142,7 +144,7 @@ class Proxy @Inject()() extends ApiController {
142144

143145
// Chunk the response
144146
val bodyStream = originalResponse.ahcResponse.getResponseBodyAsStream
145-
val bodyEnumerator = Enumerator.fromStream(bodyStream)
147+
val bodyEnumerator = Enumerator.fromStream(bodyStream, chunksize)
146148
val payload = Status(statusCode).chunked(bodyEnumerator)
147149

148150
// Return a SimpleResult, coerced into our desired Content-Type

0 commit comments

Comments
 (0)