Skip to content

Commit f348fa8

Browse files
lmarinilmarini
authored
Merge branch 'develop' into extractor-job-tracking-improvements
2 parents 3eafc6f + c7d83b1 commit f348fa8

File tree

13 files changed

+600
-170
lines changed

13 files changed

+600
-170
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 108 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,17 @@ jobs:
3131
runs-on: ubuntu-latest
3232
steps:
3333
- uses: actions/checkout@v2
34+
- name: github branch
35+
run: |
36+
BRANCH=${GITHUB_REF##*/}
37+
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
38+
if [ "$BRANCH" == "master" ]; then
39+
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
40+
elif [ "$BRANCH" == "develop" ]; then
41+
echo "::set-env name=CLOWDER_VERSION::develop"
42+
else
43+
echo "::set-env name=CLOWDER_VERSION::testing"
44+
fi
3445
- uses: actions/setup-java@v1
3546
with:
3647
java-version: 1.8
@@ -46,8 +57,18 @@ jobs:
4657
key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
4758
- name: sbt clean update
4859
run: ./sbt clean update
60+
env:
61+
BRANCH: ${{ env.GITHUB_BRANCH }}
62+
VERSION: ${{ env.CLOWDER_VERSION }}
63+
BUILDNUMBER: ${{ github.run_number }}
64+
GITSHA1: ${{ github.sha }}
4965
- name: sbt compile
5066
run: ./sbt compile
67+
env:
68+
BRANCH: ${{ env.GITHUB_BRANCH }}
69+
VERSION: ${{ env.CLOWDER_VERSION }}
70+
BUILDNUMBER: ${{ github.run_number }}
71+
GITSHA1: ${{ github.sha }}
5172

5273
# starts a mongodb instance, and runs the scala tests
5374
test:
@@ -59,10 +80,18 @@ jobs:
5980
ports:
6081
- 27017:27017
6182
steps:
62-
- uses: actions/setup-java@v1
63-
with:
64-
java-version: 1.8
6583
- uses: actions/checkout@v2
84+
- name: github branch
85+
run: |
86+
BRANCH=${GITHUB_REF##*/}
87+
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
88+
if [ "$BRANCH" == "master" ]; then
89+
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
90+
elif [ "$BRANCH" == "develop" ]; then
91+
echo "::set-env name=CLOWDER_VERSION::develop"
92+
else
93+
echo "::set-env name=CLOWDER_VERSION::testing"
94+
fi
6695
- uses: actions/setup-java@v1
6796
with:
6897
java-version: 1.8
@@ -78,6 +107,11 @@ jobs:
78107
key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
79108
- name: sbt test
80109
run: ./sbt "test-only integration.APITestSuite"
110+
env:
111+
BRANCH: ${{ env.GITHUB_BRANCH }}
112+
VERSION: ${{ env.CLOWDER_VERSION }}
113+
BUILDNUMBER: ${{ github.run_number }}
114+
GITSHA1: ${{ github.sha }}
81115

82116
# creates zip file of the dist compiled version. The results are
83117
# uploaded as artifacts for this build as well to the release if
@@ -87,6 +121,17 @@ jobs:
87121
needs: build
88122
steps:
89123
- uses: actions/checkout@v2
124+
- name: github branch
125+
run: |
126+
BRANCH=${GITHUB_REF##*/}
127+
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
128+
if [ "$BRANCH" == "master" ]; then
129+
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
130+
elif [ "$BRANCH" == "develop" ]; then
131+
echo "::set-env name=CLOWDER_VERSION::develop"
132+
else
133+
echo "::set-env name=CLOWDER_VERSION::testing"
134+
fi
90135
- uses: actions/setup-java@v1
91136
with:
92137
java-version: 1.8
@@ -102,20 +147,34 @@ jobs:
102147
key: ${{ runner.os }}-sbt-${{ hashFiles('project/Build.scala') }}
103148
- name: sbt dist
104149
run: ./sbt dist
150+
env:
151+
BRANCH: ${{ env.GITHUB_BRANCH }}
152+
VERSION: ${{ env.CLOWDER_VERSION }}
153+
BUILDNUMBER: ${{ github.run_number }}
154+
GITSHA1: ${{ github.sha }}
105155
- uses: actions/upload-artifact@v2
106156
with:
107157
name: clowder.zip
108158
path: target/universal/clowder-*.zip
109159
- name: Upload files to a GitHub release
110-
uses: svenstaro/[email protected]
111160
if: github.event_name == 'release' && github.event.action == 'created'
161+
uses: svenstaro/[email protected]
112162
with:
113163
repo_token: ${{ secrets.GITHUB_TOKEN }}
114164
tag: ${{ github.ref }}
115165
overwrite: true
116166
asset_name: clowder.zip
117167
file: target/universal/clowder-*.zip
118168
file_glob: true
169+
- name: Upload dist to NCSA
170+
if: github.event_name == 'push'
171+
uses: robkooper/sftp-action@master
172+
with:
173+
host: ${{ secrets.SCP_HOST }}
174+
username: ${{ secrets.SCP_USERNAME }}
175+
key: ${{ secrets.SCP_KEY }}
176+
files: "target/universal/clowder-*.zip"
177+
target: "CATS/${{ env.CLOWDER_VERSION }}/files"
119178

120179
# creates scaladoc, html and epub (no pdflatex) and uploads those
121180
# as artifacts for this build as well to the release if created.
@@ -124,6 +183,17 @@ jobs:
124183
needs: build
125184
steps:
126185
- uses: actions/checkout@v2
186+
- name: github branch
187+
run: |
188+
BRANCH=${GITHUB_REF##*/}
189+
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
190+
if [ "$BRANCH" == "master" ]; then
191+
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
192+
elif [ "$BRANCH" == "develop" ]; then
193+
echo "::set-env name=CLOWDER_VERSION::develop"
194+
else
195+
echo "::set-env name=CLOWDER_VERSION::testing"
196+
fi
127197
- uses: actions/setup-java@v1
128198
with:
129199
java-version: 1.8
@@ -143,11 +213,25 @@ jobs:
143213
python-version: 3.7
144214
- name: sbt doc
145215
run: ./sbt doc
216+
env:
217+
BRANCH: ${{ env.GITHUB_BRANCH }}
218+
VERSION: ${{ env.CLOWDER_VERSION }}
219+
BUILDNUMBER: ${{ github.run_number }}
220+
GITSHA1: ${{ github.sha }}
146221
- uses: actions/upload-artifact@v2
147222
with:
148223
name: ScalaDoc
149224
path: target/scala-*/api/
150-
- name: sphinx
225+
- name: Upload scaladoc to NCSA
226+
if: github.event_name == 'push'
227+
uses: robkooper/sftp-action@master
228+
with:
229+
host: ${{ secrets.SCP_HOST }}
230+
username: ${{ secrets.SCP_USERNAME }}
231+
key: ${{ secrets.SCP_KEY }}
232+
files: "target/scala-*/api/*"
233+
target: "CATS/${{ env.CLOWDER_VERSION }}/documentation/scaladoc"
234+
- name: sphinx
151235
run: |
152236
cd doc/src/sphinx/
153237
python -m pip install -r requirements.txt
@@ -156,16 +240,34 @@ jobs:
156240
with:
157241
name: HTML Documentation
158242
path: doc/src/sphinx/_build/html
243+
- name: Upload sphinx to NCSA
244+
if: github.event_name == 'push'
245+
uses: robkooper/sftp-action@master
246+
with:
247+
host: ${{ secrets.SCP_HOST }}
248+
username: ${{ secrets.SCP_USERNAME }}
249+
key: ${{ secrets.SCP_KEY }}
250+
files: "doc/src/sphinx/_build/html/*"
251+
target: "CATS/${{ env.CLOWDER_VERSION }}/documentation/sphinx"
159252
- uses: actions/upload-artifact@v2
160253
with:
161254
name: EPUB Documentation
162255
path: doc/src/sphinx/_build/epub/Clowder.epub
163256
- name: Upload files to a GitHub release
164-
uses: svenstaro/[email protected]
165257
if: github.event_name == 'release' && github.event.action == 'created'
258+
uses: svenstaro/[email protected]
166259
with:
167260
repo_token: ${{ secrets.GITHUB_TOKEN }}
168261
tag: ${{ github.ref }}
169262
overwrite: true
170263
asset_name: clowder.epub
171264
file: doc/src/sphinx/_build/epub/Clowder.epub
265+
- name: Upload epub to NCSA
266+
if: github.event_name == 'push'
267+
uses: robkooper/sftp-action@master
268+
with:
269+
host: ${{ secrets.SCP_HOST }}
270+
username: ${{ secrets.SCP_USERNAME }}
271+
key: ${{ secrets.SCP_KEY }}
272+
files: "doc/src/sphinx/_build/epub/Clowder.epub"
273+
target: "CATS/${{ env.CLOWDER_VERSION }}/files"

‎.github/workflows/docker.yml‎

Lines changed: 53 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
name: Docker
22

3+
# initially we would us on: [release] as well, the problem is that
4+
# the code in clowder would not know what branch the code is in,
5+
# and would not set the right version flags.
6+
37
# This will run when:
4-
# - a new release is created, to make sure the right tags of the
5-
# docker images are pushed (expects tags to be v1.8.4).
68
# - when new code is pushed to master/develop to push the tags
79
# latest and develop
810
# - when a pull request is created and updated to make sure the
@@ -12,9 +14,6 @@ name: Docker
1214
# - DOCKERHUB_USERNAME : username that can push to the org
1315
# - DOCKERHUB_PASSWORD : password asscoaited with the username
1416
on:
15-
release:
16-
types: created
17-
1817
push:
1918
branches:
2019
- master
@@ -63,69 +62,98 @@ jobs:
6362
steps:
6463
- uses: actions/checkout@v2
6564

65+
# calculate some variables that are used later
66+
- name: github branch
67+
run: |
68+
BRANCH=${GITHUB_REF##*/}
69+
echo "::set-env name=GITHUB_BRANCH::${BRANCH}"
70+
if [ "$BRANCH" == "master" ]; then
71+
version="$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
72+
tags="latest"
73+
oldversion=""
74+
while [ "${oldversion}" != "${version}" ]; do
75+
oldversion="${version}"
76+
tags="${tags},${version}"
77+
version=${version%.*}
78+
done
79+
echo "::set-env name=CLOWDER_VERSION::$(awk '/version = / { print $4 }' project/Build.scala | sed 's/"//g')"
80+
echo "::set-env name=CLOWDER_TAGS::${tags}"
81+
elif [ "$BRANCH" == "develop" ]; then
82+
echo "::set-env name=CLOWDER_VERSION::develop"
83+
echo "::set-env name=CLOWDER_TAGS::develop"
84+
else
85+
echo "::set-env name=CLOWDER_VERSION::testing"
86+
echo "::set-env name=CLOWDER_TAGS::"
87+
fi
88+
6689
# build the docker image, this will always run to make sure
6790
# the Dockerfile still works.
6891
- name: Build image
6992
run: |
7093
docker build \
71-
--build-arg BRANCH=${{ github.ref }} \
72-
--build-arg VERSION=${{ github.run_number }} \
94+
--build-arg BRANCH=${GITHUB_BRANCH} \
95+
--build-arg VERSION=${CLOWDER_VERSION} \
7396
--build-arg BUILDNUMBER=${{ github.run_number }} \
7497
--build-arg GITSHA1=${{ github.sha }} \
7598
--tag image ${{ matrix.FOLDER }}
7699
77100
# this will publish to the actor (person) github packages
78101
- name: Publish to GitHub
79-
if: github.event_name != 'pull_request'
80-
uses: elgohr/Publish-Docker-Github-Action@2.14
102+
if: github.event_name == 'push'
103+
uses: elgohr/Publish-Docker-Github-Action@2.18
81104
env:
82-
BRANCH: ${{ github.ref }}
83-
VERSION: ${{ github.run_number }}
105+
BRANCH: ${{ env.GITHUB_BRANCH }}
106+
VERSION: ${{ env.CLOWDER_VERSION }}
84107
BUILDNUMBER: ${{ github.run_number }}
85108
GITSHA1: ${{ github.sha }}
86109
with:
87110
name: ${{ github.repository_owner }}/clowder/${{ matrix.IMAGE }}
88111
username: ${{ github.actor }}
89112
password: ${{ secrets.GITHUB_TOKEN }}
90113
context: ${{ matrix.FOLDER }}
91-
tag_semver: true
92-
tag_names: true
114+
tags: "${{ env.CLOWDER_TAGS }}"
93115
registry: docker.pkg.github.com
94116
buildargs: BRANCH,VERSION,BUILDNUMBER,GITSHA1
95117

96118
# this will publish to the clowder dockerhub repo
97119
- name: Publish to Docker Hub
98-
if: github.event_name != 'pull_request' && github.repository == env.MASTER_REPO
99-
uses: elgohr/Publish-Docker-Github-Action@2.14
120+
if: github.event_name == 'push' && github.repository == env.MASTER_REPO
121+
uses: elgohr/Publish-Docker-Github-Action@2.18
100122
env:
101-
BRANCH: ${{ github.ref }}
102-
VERSION: ${{ github.run_number }}
123+
BRANCH: ${{ env.GITHUB_BRANCH }}
124+
VERSION: ${{ env.CLOWDER_VERSION }}
103125
BUILDNUMBER: ${{ github.run_number }}
104126
GITSHA1: ${{ github.sha }}
105127
with:
106128
name: ${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}
107129
username: ${{ secrets.DOCKERHUB_USERNAME }}
108130
password: ${{ secrets.DOCKERHUB_PASSWORD }}
109131
context: ${{ matrix.FOLDER }}
110-
tag_semver: true
111-
tag_names: true
132+
tags: "${{ env.CLOWDER_TAGS }}"
112133
buildargs: BRANCH,VERSION,BUILDNUMBER,GITSHA1
113134

114135
# this will update the README of the dockerhub repo
115136
- name: check file
116137
id: filecheck
117-
if: github.event_name == 'release' && github.repository == env.MASTER_REPO
118138
run: |
119-
if [ "${{ matrix.README }}" != "" -a -e "${{ matrix.README }}" ]; then
120-
echo "##[set-output name=exists;]true"
139+
if [ "${{ matrix.README }}" != "" ]; then
140+
if [ -e "${{ matrix.README }}" ]; then
141+
echo "##[set-output name=readme;]${{ matrix.README }}"
142+
else
143+
echo "##[set-output name=readme;]"
144+
fi
121145
else
122-
echo "##[set-output name=exists;]false"
146+
if [ -e "${{ matrix.FOLDER }}/README.md" ]; then
147+
echo "##[set-output name=readme;]${{ matrix.FOLDER }}/README.md"
148+
else
149+
echo "##[set-output name=readme;]"
150+
fi
123151
fi
124152
- name: Docker Hub Description
125-
if: github.event_name == 'release' && github.repository == env.MASTER_REPO && steps.filecheck.outputs.exists == 'true'
153+
if: github.event_name == 'push' && github.repository == env.MASTER_REPO && env.BRANCH == 'master' && steps.filecheck.outputs.readme != ''
126154
uses: peter-evans/dockerhub-description@v2
127155
env:
128156
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
129157
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
130158
DOCKERHUB_REPOSITORY: ${{ env.DOCKERHUB_ORG }}/${{ matrix.IMAGE }}
131-
README_FILEPATH: ${{ matrix.README }}
159+
README_FILEPATH: ${{ steps.filecheck.outputs.readme }}

‎CHANGELOG.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## 1.10.1 - 2020-07-16
8+
9+
### Fixed
10+
- Queue threads (e.g. Elasticsearch indexer) will no longer crash permanently if the queue connection to Mongo is lost temporarily.
11+
- Docker images would not build correctly on GitHub.
12+
- If monitor HTTP server would crash, it would not restart correctly.
13+
- Don't call server side twice when rendering list of files on dataset page.
14+
[#7](https://github.com/clowder-framework/clowder/issues/7)
15+
- Fixed Sphinx build errors and switched to using pipenv. Now building docs on [readthedocs](https://clowder-framework.readthedocs.io/en/latest/).
16+
17+
### Added
18+
- GitHub artifacts can be uploaded using SCP to remote server.
19+
720
## 1.10.0 - 2020-06-30
821

922
### Added

‎app/services/RabbitmqPlugin.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1239,7 +1239,7 @@ class ExtractorsHeartbeats(channel: Channel, queue: String) extends Actor {
12391239
// Update database
12401240
extractionInfoResult.fold(
12411241
errors => {
1242-
Logger.error("Received extractor heartbeat with bad format: " + extractor_info)
1242+
Logger.debug("Received extractor heartbeat with bad format: " + extractor_info)
12431243
},
12441244
info => {
12451245
extractorsService.getExtractorInfo(info.name) match {

0 commit comments

Comments
 (0)