Skip to content

Commit e591226

Browse files
robkooperrobkooper
committed
Merge remote-tracking branch 'origin/master' into pypi
2 parents d6282dd + 0ccb930 commit e591226

File tree

11 files changed

+256
-142
lines changed

11 files changed

+256
-142
lines changed

‎.zenodo.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{
2+
"title": "Python wrappers for use with the Clowder system",
3+
"description": "This library makes it easier to interact with clowder and to create extractors as well as code that will interact with clowder. The library will encapsulate the calls to the REST API. For extractors the library will provide the developer with an easy to extend framework to create new extractors.",
4+
"creators": [
5+
{
6+
"affiliation": "National Center for Supercomputing Applications",
7+
"name": "Rob Kooper",
8+
"orcid": "0000-0002-5781-7287"
9+
},
10+
{
11+
"affiliation": "National Center for Supercomputing Applications",
12+
"name": "Max Burnette"
13+
},
14+
{
15+
"affiliation": "National Center for Supercomputing Applications",
16+
"name": "Sandeep Satheesan"
17+
},
18+
{
19+
"affiliation": "National Center for Supercomputing Applications",
20+
"name": "Bing Zhang"
21+
},
22+
{
23+
"affiliation": "University of Illinois at Urbana-Champaign",
24+
"name": "Todd Nicholson"
25+
},
26+
{
27+
"affiliation": "National Center for Supercomputing Applications",
28+
"name": "Indira Gutierrez"
29+
},
30+
{
31+
"affiliation": "National Center for Supercomputing Applications",
32+
"name": "Kenton McHenry"
33+
},
34+
{
35+
"name": "Ward Poelmans"
36+
}
37+
],
38+
"upload_type": "software",
39+
"license": "NCSA",
40+
"access_right": "open",
41+
"keywords": [
42+
"json",
43+
"python",
44+
"clowder"
45+
],
46+
"notes": "Development is supported by Brown Dog (NSF #1261582)"
47+
}

‎Dockerfile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
FROM ubuntu:16.04
22
MAINTAINER Rob Kooper <[email protected]>
33

4+
# environment variables
5+
ENV RABBITMQ_URI="" \
6+
RABBITMQ_EXCHANGE="clowder" \
7+
RABBITMQ_QUEUE="" \
8+
REGISTRATION_ENDPOINTS="https://clowder.ncsa.illinois.edu/extractors" \
9+
MAIN_SCRIPT=""
10+
11+
# install python
412
RUN apt-get -q -q update && apt-get install -y --no-install-recommends \
513
netcat \
614
python \
@@ -9,9 +17,13 @@ RUN apt-get -q -q update && apt-get install -y --no-install-recommends \
917
&& rm -rf /var/lib/apt/lists/* \
1018
&& adduser --system clowder
1119

20+
# instal pyclowder2
1221
COPY pyclowder /tmp/pyclowder/pyclowder
1322
COPY setup.py requirements.txt /tmp/pyclowder/
1423

1524
RUN pip install --upgrade -r /tmp/pyclowder/requirements.txt \
1625
&& pip install --upgrade /tmp/pyclowder \
1726
&& rm -rf /tmp/pyclowder
27+
28+
# change folder
29+
WORKDIR /home/clowder/

‎Dockerfile.onbuild‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM clowder/pyclowder:2
2+
3+
# copy all files
4+
ONBUILD ADD . /home/clowder/
5+
6+
# install any packages
7+
#ONBUILD COPY packages.apt /home/clowder/
8+
ONBUILD RUN if [ -e packages.apt ]; then \
9+
apt-get -q -q update \
10+
&& xargs apt-get -y install --no-install-recommends < packages.apt \
11+
&& rm -rf /var/lib/apt/lists/*; \
12+
fi
13+
14+
# install any python packages
15+
#ONBUILD COPY requirements.txt /home/clowder/
16+
ONBUILD RUN if [ -e requirements.txt ]; then \
17+
pip install --no-cache-dir -r requirements.txt; \
18+
fi
19+
20+
# switch to user clowder last minute
21+
ONBUILD USER clowder
22+
23+
# command to run when starting container
24+
COPY entrypoint.sh /home/clowder/
25+
ENTRYPOINT ["/home/clowder/entrypoint.sh"]
26+
CMD ["extractor"]

‎README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
[![DOI](https://zenodo.org/badge/126513159.svg)](https://zenodo.org/badge/latestdoi/126513159)
2+
13
This repository contains the next generation of pyClowder. This library makes it easier to interact with clowder and
24
to create extractors.
35

@@ -8,6 +10,21 @@ created using extractors. To make it easy to create these extractors in python w
810
Besides wrapping often used api calls in convenient python calls, we have also added some code to make it easy to
911
create new extractors.
1012

13+
## Setup
14+
15+
Install pyClowder2 on your system by cloning this repo:
16+
17+
```
18+
git clone https://opensource.ncsa.illinois.edu/bitbucket/scm/cats/pyclowder2.git
19+
cd pyclowder2
20+
pip install -r requirements.txt
21+
python setup.py install
22+
```
23+
or directly from Bitbucket:
24+
```
25+
pip install -r https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS/repos/pyclowder2/raw/requirements.txt git+https://opensource.ncsa.illinois.edu/bitbucket/scm/cats/pyclowder2.git
26+
```
27+
1128
## Example Extractor
1229

1330
Following is an example of the WordCount extractor. This example will allow the user to specify from the command line

‎docker.sh‎

Lines changed: 9 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,89 +1,17 @@
11
#!/bin/sh
22

3-
# variables that can be set
4-
# DEBUG : set to echo to print command and not execute
5-
# PUSH : set to push to push, anthing else not to push. If not set
6-
# the program will push if master or develop.
7-
# PROJECT : the project to add to the image, default is NCSA
8-
# VERSION : the list of tags to use, if not set this will be 2
9-
103
#DEBUG=echo
114

12-
# set default for clowder
13-
PROJECT=${PROJECT:-"clowder"}
14-
15-
RM=${RM:-"rm"}
16-
17-
# find out version and if we should push
18-
BRANCH="$(git rev-parse --abbrev-ref HEAD)"
19-
VERSION=${VERSION:-"2"}
20-
if [ "$BRANCH" = "master" ]; then
21-
PUSH=${PUSH:-"push"}
22-
else
23-
PUSH=${PUSH:-""}
24-
fi
25-
26-
# keep track of which latest amde
27-
LATEST=""
28-
29-
# helper to create the docker container
30-
# $1 - folder that contains the Dockerfile
31-
# $2 - name of docker image
32-
# $3 - name of Dockerfile
33-
create() {
34-
if [ -z "$1" ]; then echo "Missing repo/Dockerfile name."; exit -1; fi
35-
if [ -z "$2" ]; then echo "Missing name for $1."; exit -1; fi
36-
37-
DOCKERFILE=${3:-"$1/Dockerfile"}
38-
39-
# create image using temp id
40-
local ID=$(uuidgen)
41-
${DEBUG} docker build --tag $$ --file ${DOCKERFILE} $1
42-
if [ $? -ne 0 ]; then
43-
echo "FAILED build of $1/${DOCKERFILE}"
44-
exit -1
45-
fi
46-
47-
# tag all versions
48-
for v in $VERSION; do
49-
if [ "$PROJECT" = "" ]; then
50-
${DEBUG} docker tag $$ ${2}:${v}
51-
else
52-
for p in ${PROJECT}; do
53-
NAME=$2
54-
${DEBUG} docker tag $$ ${p}/${NAME}:${v}
55-
if [ "$PUSH" = "push" ]; then
56-
${DEBUG} docker push ${p}/${NAME}:${v}
57-
fi
58-
done
59-
fi
60-
done
61-
62-
# tag version as latest, but don't push
63-
if [ ! "$BRANCH" = "master" ]; then
64-
if [ "$PROJECT" = "" ]; then
65-
${DEBUG} docker tag $$ ${2}:latest
66-
LATEST="$LATEST ${2}:latest"
67-
else
68-
for p in ${PROJECT}; do
69-
NAME=$2
70-
${DEBUG} docker tag $$ ${p}/${NAME}:latest
71-
LATEST="$LATEST ${p}/${NAME}:latest"
72-
done
73-
fi
74-
fi
5+
# build docker container
6+
${DEBUG} docker build --tag clowder/pyclowder:2 .
7+
${DEBUG} docker build --tag clowder/pyclowder:onbuild --file Dockerfile.onbuild .
758

76-
# delete image with temp id
77-
${DEBUG} docker rmi $$
78-
}
9+
# build sample extractors
10+
${DEBUG} docker build --tag clowder/extractors-wordcount:2 sample-extractors/wordcount
7911

80-
# Create the docker containers
81-
create "." "pyclowder"
82-
create "sample-extractors/wordcount" "extractors-wordcount"
8312

84-
# remove latest tags
85-
if [ "$RM" = "rm" ]; then
86-
for r in $LATEST; do
87-
${DEBUG} docker rmi ${r}
88-
done
13+
if [ "$(git rev-parse --abbrev-ref HEAD)" = "master" ]; then
14+
${DEBUG} docker push clowder/pyclowder:2
15+
${DEBUG} docker push clowder/pyclowder:onbuild
16+
${DEBUG} docker push clowder/extractors-wordcount:2
8917
fi

‎entrypoint.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# rabbitmq
5+
if [ "${RABBITMQ_URI}" == "" ]; then
6+
7+
# configure RABBITMQ_URI if started using docker-compose or --link flag
8+
if [ -n "${RABBITMQ_PORT_5672_TCP_ADDR}" ]; then
9+
RABBITMQ_URI="amqp://${RABBITMQ_PORT_5672_TCP_ADDR}:${RABBITMQ_PORT_5672_TCP_PORT}/%2F"
10+
fi
11+
fi
12+
13+
# start server if asked
14+
if [ "$1" = 'extractor' ]; then
15+
# make sure main script exists
16+
if [ "${MAIN_SCRIPT}" == "" ]; then
17+
echo "No main script specified, can not run code."
18+
exit -1
19+
fi
20+
if [ ! -e "${MAIN_SCRIPT}" ]; then
21+
echo "Main script specified does not exist."
22+
exit -1
23+
fi
24+
chmod 755 "${MAIN_SCRIPT}"
25+
26+
# check to make sure rabbitmq is up
27+
if [ "${RABBITMQ_PORT_5672_TCP_ADDR}" != "" ]; then
28+
# start extractor after rabbitmq is up
29+
for i in `seq 1 10`; do
30+
if nc -z ${RABBITMQ_PORT_5672_TCP_ADDR} ${RABBITMQ_PORT_5672_TCP_PORT} ; then
31+
break
32+
fi
33+
sleep 1
34+
done
35+
fi
36+
37+
# launch extractor and see what happens
38+
exec "./${MAIN_SCRIPT}"
39+
fi
40+
41+
exec "$@"

‎pyclowder/collections.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import logging
88
import requests
99

10+
from pyclowder.client import ClowderClient
1011
from pyclowder.utils import StatusMessage
1112

1213

@@ -141,3 +142,73 @@ def upload_preview(connector, host, key, collectionid, previewfile, previewmetad
141142
result.raise_for_status()
142143

143144
return previewid
145+
146+
147+
class CollectionsApi(object):
148+
"""
149+
API to manage the REST CRUD endpoints for collections
150+
"""
151+
def __init__(self, client=None, host=None, key=None, username=None, password=None):
152+
"""Set client if provided otherwise create new one"""
153+
if client:
154+
self.api_client = client
155+
else:
156+
self.client = ClowderClient(host=host, key=key, username=username, password=password)
157+
158+
def create(self, name, description, parent_id, space_id):
159+
"""Create a new collection in Clowder.
160+
161+
Keyword arguments:
162+
connector -- connector information, used to get missing parameters and send status updates
163+
host -- the clowder host, including http and port, should end with a /
164+
key -- the secret key to login to clowder
165+
collectionname -- name of new dataset to create
166+
description -- description of new dataset
167+
parentid -- id of parent collection
168+
spaceid -- id of the space to add dataset to
169+
"""
170+
171+
if parent_id:
172+
if space_id:
173+
body = {
174+
"name": name,
175+
"description": description,
176+
"parentId": [parent_id],
177+
"space": space_id
178+
}
179+
result = self.client.post("/collections/newCollectionWithParents", body)
180+
else:
181+
body = {
182+
"name": name,
183+
"description": description,
184+
"parentId": [parent_id],
185+
}
186+
result = self.client.post("/collections/newCollectionWithParent", body)
187+
else:
188+
if space_id:
189+
body = {
190+
"name": name,
191+
"description": description,
192+
"space": space_id
193+
}
194+
result = self.client.post("/collections", body)
195+
else:
196+
body = {
197+
"name": name,
198+
"description": description,
199+
}
200+
result = self.client.post("/collections", body)
201+
result.raise_for_status()
202+
203+
collection_id = result.json()['id']
204+
logging.debug("collection id = [%s]", collection_id)
205+
206+
return collection_id
207+
208+
def get_all_collections(self):
209+
"""
210+
Get All Collections in Clowder
211+
212+
:return: List of collections in Clowder
213+
"""
214+
return self.client.get("/collections")

‎pyclowder/files.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ def upload_metadata(connector, host, key, fileid, metadata):
188188

189189

190190
# pylint: disable=too-many-arguments
191-
def upload_preview(connector, host, key, fileid, previewfile, previewmetadata):
191+
def upload_preview(connector, host, key, fileid, previewfile, previewmetadata, preview_mimetype=None):
192192
"""Upload preview to Clowder.
193193
194194
Keyword arguments:
@@ -199,6 +199,8 @@ def upload_preview(connector, host, key, fileid, previewfile, previewmetadata):
199199
previewfile -- the file containing the preview
200200
previewmetadata -- any metadata to be associated with preview, can contain a section_id
201201
to indicate the section this preview should be associated with.
202+
preview_mimetype -- (optional) MIME type of the preview file. By default, this is obtained from the
203+
file itself and this parameter can be ignored. E.g. 'application/vnd.clowder+custom+xml'
202204
"""
203205

204206
connector.status_update(StatusMessage.processing, {"type": "file", "id": fileid}, "Uploading file preview.")
@@ -209,7 +211,13 @@ def upload_preview(connector, host, key, fileid, previewfile, previewmetadata):
209211
# upload preview
210212
url = '%sapi/previews?key=%s' % (host, key)
211213
with open(previewfile, 'rb') as filebytes:
212-
result = connector.post(url, files={"File": filebytes}, verify=connector.ssl_verify if connector else True)
214+
# If a custom preview file MIME type is provided, use it to generate the preview file object.
215+
if preview_mimetype is not None:
216+
result = connector.post(url, files={"File": (os.path.basename(previewfile), filebytes, preview_mimetype)},
217+
verify=connector.ssl_verify if connector else True)
218+
else:
219+
result = connector.post(url, files={"File": filebytes}, verify=connector.ssl_verify if connector else True)
220+
213221
previewid = result.json()['id']
214222
logger.debug("preview id = [%s]", previewid)
215223

0 commit comments

Comments
 (0)