Skip to content

Commit 398b975

Browse files
robkooperrobkooper
committed
Merge pull request #65 in CATS/pyclowder2 from CATS-872-core-extractors-need-to-be-ported-to-pyclowder-2 to develop
* commit '64122109cec2f510038417a4ec0c3b445cfb4aff': cleanup docker builds - added a simple extractor that works on binary files and creates image thumbnail, image preview and preview - docker.sh simplified now to be used when pushing tags/releases - split docker.sh into release.sh and docker.sh. docker.sh will build docker containers, release.sh will do the actual push to docker hub.
2 parents aec169a + 6412210 commit 398b975

File tree

9 files changed

+275
-100
lines changed

9 files changed

+275
-100
lines changed

‎Dockerfile‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,10 @@ ENV RABBITMQ_URI="amqp://guest:guest@rabbitmq:5672/%2F" \
99

1010
# install python
1111
RUN apt-get -q -q update && apt-get install -y --no-install-recommends \
12-
netcat \
1312
python \
1413
python-pip \
1514
&& pip install --upgrade setuptools \
16-
&& rm -rf /var/lib/apt/lists/* \
17-
&& adduser --system clowder
15+
&& rm -rf /var/lib/apt/lists/*
1816

1917
# instal pyclowder2
2018
COPY pyclowder /tmp/pyclowder/pyclowder
@@ -23,5 +21,8 @@ COPY setup.py description.rst /tmp/pyclowder/
2321
RUN pip install --upgrade /tmp/pyclowder \
2422
&& rm -rf /tmp/pyclowder
2523

26-
# change folder
27-
WORKDIR /home/clowder/
24+
# folder for pyclowder code
25+
WORKDIR /home/clowder
26+
27+
# command to run when starting container
28+
CMD python "./${MAIN_SCRIPT}"

‎Dockerfile.onbuild‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM clowder/pyclowder:2
1+
FROM clowder/pyclowder
22

33
# install any packages
44
ONBUILD COPY packages.* Dockerfile /home/clowder/
@@ -16,11 +16,3 @@ ONBUILD RUN if [ -e requirements.txt ]; then \
1616

1717
# copy all files
1818
ONBUILD ADD . /home/clowder/
19-
20-
# switch to user clowder last minute
21-
ONBUILD USER clowder
22-
23-
# command to run when starting container
24-
COPY entrypoint.sh /home/clowder/
25-
ENTRYPOINT ["/home/clowder/entrypoint.sh"]
26-
CMD ["extractor"]

‎docker-compose.yml‎

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎docker.sh‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
#!/bin/sh
22

3-
#DEBUG=echo
3+
# exit on error, with error code
4+
set -e
5+
6+
# use DEBUG=echo ./release.sh to print all commands
7+
export DEBUG=${DEBUG:-""}
48

59
# build docker container
6-
${DEBUG} docker build --tag clowder/pyclowder:2 .
7-
${DEBUG} docker build --tag clowder/pyclowder:onbuild --file Dockerfile.onbuild .
10+
${DEBUG} docker build --tag clowder/pyclowder:latest .
11+
${DEBUG} docker build --tag clowder/pyclowder:onbuild --file Dockerfile.onbuild .
12+
${DEBUG} docker build --tag clowder/extractors-binary-preview:onbuild sample-extractors/binary-preview
813

914
# build sample extractors
10-
${DEBUG} docker build --tag clowder/extractors-wordcount:2 sample-extractors/wordcount
11-
12-
13-
if [ "$(git rev-parse --abbrev-ref HEAD)" = "master" ]; then
14-
${DEBUG} docker push clowder/pyclowder:2
15-
${DEBUG} docker push clowder/pyclowder:onbuild
16-
${DEBUG} docker push clowder/extractors-wordcount:2
17-
fi
15+
${DEBUG} docker build --tag clowder/extractors-wordcount:latest sample-extractors/wordcount

‎entrypoint.sh‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎release.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
# exit on error, with error code
4+
set -e
5+
6+
# can use the following to push to isda-registry for testing:
7+
# BRANCH="master" SERVER=isda-registry.ncsa.illinois.edu/ ./release.sh
8+
9+
# use DEBUG=echo ./release.sh to print all commands
10+
export DEBUG=${DEBUG:-""}
11+
12+
# use SERVER=XYZ/ to push to a different server
13+
SERVER=${SERVER:-""}
14+
15+
# what branch are we on
16+
BRANCH=${BRANCH:-"$(git rev-parse --abbrev-ref HEAD)"}
17+
18+
# make sure docker is build
19+
$(dirname $0)/docker.sh
20+
21+
# check branch and set version
22+
if [ "${BRANCH}" == "master" ]; then
23+
VERSION=${VERSION:-"2.0.2 2.0 2 latest"}
24+
elif [ "${BRANCH}" == "develop" ]; then
25+
VERSION="develop"
26+
else
27+
exit 0
28+
fi
29+
30+
# tag all images and push if needed
31+
for i in pyclowder extractors-wordcount; do
32+
for v in ${VERSION}; do
33+
if [ "$v" != "latest" -o "$SERVER" != "" ]; then
34+
${DEBUG} docker tag clowder/${i}:latest ${SERVER}clowder/${i}:${v}
35+
fi
36+
${DEBUG} docker push ${SERVER}clowder/${i}:${v}
37+
done
38+
done
39+
40+
for i in pyclowder extractors-binary-preview; do
41+
for v in ${VERSION}; do
42+
if [ "$v" != "latest" ]; then
43+
${DEBUG} docker tag clowder/${i}:onbuild ${SERVER}clowder/${i}:${v}-onbuild
44+
${DEBUG} docker push ${SERVER}clowder/${i}:${v}-onbuild
45+
elif [ "$SERVER" != "" ]; then
46+
${DEBUG} docker tag clowder/${i}:onbuild ${SERVER}clowder/${i}:onbuild
47+
${DEBUG} docker push ${SERVER}clowder/${i}:onbuild
48+
else
49+
${DEBUG} docker push clowder/${i}:onbuild
50+
fi
51+
done
52+
done

‎sample-extractors/binary-preview/Dockerfile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM clowder/pyclowder:onbuild
2+
3+
ENV MAIN_SCRIPT="binary_extractor.py" \
4+
RABBITMQ_QUEUE="" \
5+
IMAGE_BINARY="" \
6+
IMAGE_TYPE="" \
7+
IMAGE_THUMBNAIL_COMMAND="" \
8+
IMAGE_PREVIEW_COMMAND="" \
9+
PREVIEW_BINARY="" \
10+
PREVIEW_TYPE="" \
11+
PREVIEW_COMMAND=""
12+
13+
ONBUILD COPY packages.* Dockerfile /home/clowder/
14+
ONBUILD RUN if [ -e packages.apt ]; then \
15+
apt-get -q -q update \
16+
&& xargs apt-get -y install --no-install-recommends < packages.apt \
17+
&& rm -rf /var/lib/apt/lists/*; \
18+
fi
19+
20+
ONBUILD COPY extractor_info.json /home/clowder/

‎sample-extractors/binary-preview/README.rst‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
A simple extractor that can execute binary programs to generate image previews, thumbnails, as well as previews. To
2+
use this extractor you will need to create a new Dockerfile that sets environment variables, a packages.apt file that
3+
lists all the packages that need to be installed, and an extractor_info.json that describes the extractor.
4+
5+
For example the following files will create an extractor that can handle audio files. To build the extractor you can
6+
use `docker build -t clowder/extractor-audio`. You can now use the extractor in your clowder environment. For example
7+
if you used the docker-compose file to start clowder yo can add this extractor to the running stack using
8+
`docker run --rm --network clowder --link rabbitmq --link clowder clowder/extractor-audio` when you now upload an
9+
audio file it will automatically create a thumbnail, image preview and audio preview.
10+
11+
Dockerfile
12+
13+
.. code-block:: Dockerfile
14+
15+
FROM clowder/pyclowder:onbuild
16+
17+
ENV RABBITMQ_QUEUE="ncsa.audio.preview" \
18+
IMAGE_BINARY="/usr/bin/sox" \
19+
IMAGE_TYPE="png" \
20+
IMAGE_THUMBNAIL_COMMAND="@BINARY@ --magic @INPUT@ -n spectrogram -r -x 225 -y 200 -o @OUTPUT@" \
21+
IMAGE_PREVIEW_COMMAND="@BINARY@ --magic @INPUT@ -n spectrogram -x 800 -Y 600 -o @OUTPUT@" \
22+
PREVIEW_BINARY="/usr/bin/sox" \
23+
PREVIEW_TYPE="mp3" \
24+
PREVIEW_COMMAND="@BINARY@ --magic @INPUT@ @OUTPUT@"
25+
26+
27+
packages.apt
28+
29+
.. code-block::
30+
31+
libsox-fmt-mp3 sox
32+
33+
extractor_info.json
34+
35+
.. code-block:: json
36+
37+
{
38+
"@context":"http://clowder.ncsa.illinois.edu/contexts/extractors.jsonld",
39+
"name":"ncsa.audio.preview",
40+
"version":"1.0",
41+
"description":"Creates thumbnail and image previews of Audio files.",
42+
"author":"Rob Kooper <[email protected]>",
43+
"contributors":[],
44+
"contexts":[],
45+
"repository":[
46+
{
47+
"repType":"git",
48+
"repUrl":"https://opensource.ncsa.illinois.edu/bitbucket/scm/cats/extractors-core.git"
49+
},
50+
{
51+
"repType":"docker",
52+
"repUrl":"clowder/extractors-audio-preview"
53+
}
54+
],
55+
"external_services":[],
56+
"process":{
57+
"file":[
58+
"audio/*"
59+
]
60+
},
61+
"dependencies":[
62+
"sox-fmt-mp3",
63+
"sox"
64+
],
65+
"bibtex":[]
66+
}

0 commit comments

Comments
 (0)