Skip to content

Commit 9ed301c

Browse files
committed
Update to v3, refactor ClowderClient, example implementation in datasets
1 parent 4c27a99 commit 9ed301c

File tree

9 files changed

+23
-317
lines changed

9 files changed

+23
-317
lines changed

CHANGELOG.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## 3.0.0 - 2022-12-16
8+
This version adds Clowder 2 support and removes the old method of extractor registration in favor of reliance on heartbeats.
9+
10+
### Added
11+
- api/v1 and api/v2 code split for back compatibility as v2 is introduced.
12+
- new simplified ClowderClient is used in the new split endpoints for future refactoring.
13+
14+
### Removed
15+
- remove RABBITMQ_EXCHANGE parameter and REGISTRATION_URL parameter.
16+
- remove DatasetsAPI and object-oriented ClowderClient.
17+
718
## 2.6.0 - 2022-06-14
819

920
This will change how clowder sees the extractors. If you have an extractor, and you specify

Dockerfile

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ ENV PYTHON_VERSION=${PYTHON_VERSION} \
88
RABBITMQ_URI="amqp://guest:guest@rabbitmq:5672/%2F" \
99
RABBITMQ_QUEUE="" \
1010
CLOWDER_URL="" \
11-
REGISTRATION_ENDPOINTS="" \
1211
EMAIL_SERVER="" \
1312
EMAIL_SENDER="extractor" \
1413
MAIN_SCRIPT=""

pyclowder/api/v1/datasets.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from pyclowder.utils import StatusMessage
1212

1313

14-
def create_empty(connector, host, key, datasetname, description, parentid=None, spaceid=None):
14+
def create_empty(connector, client, datasetname, description, parentid=None, spaceid=None):
1515
"""Create a new dataset in Clowder.
1616
1717
Keyword arguments:
@@ -25,7 +25,7 @@ def create_empty(connector, host, key, datasetname, description, parentid=None,
2525
"""
2626
logger = logging.getLogger(__name__)
2727

28-
url = '%sapi/datasets/createempty?key=%s' % (host, key)
28+
url = '%sapi/datasets/createempty?key=%s' % (client.host, client.key)
2929

3030
if parentid:
3131
if spaceid:

pyclowder/api/v1/metadata.py

Whitespace-only changes.

pyclowder/api/v2/datasets.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,12 @@
1515
from pyclowder.utils import StatusMessage
1616

1717

18-
def create_empty(connector, host, key, datasetname, description, parentid=None, spaceid=None):
18+
def create_empty(connector, client, datasetname, description, parentid=None, spaceid=None):
1919
"""Create a new dataset in Clowder.
2020
2121
Keyword arguments:
2222
connector -- connector information, used to get missing parameters and send status updates
23-
host -- the clowder host, including http and port, should end with a /
24-
key -- the secret key to login to clowder
23+
client -- ClowderClient containing authentication credentials
2524
datasetname -- name of new dataset to create
2625
description -- description of new dataset
2726
parentid -- id of parent collection
@@ -30,9 +29,9 @@ def create_empty(connector, host, key, datasetname, description, parentid=None,
3029

3130
logger = logging.getLogger(__name__)
3231

33-
url = '%sapi/v2/datasets' % (host)
32+
url = '%sapi/v2/datasets' % client.host
3433
headers = {"Content-Type": "application/json",
35-
"Authorization": "Bearer " + key}
34+
"Authorization": "Bearer " + client.key}
3635
result = requests.post(url, headers=headers,
3736
data=json.dumps({"name": datasetname, "description": description}),
3837
verify=connector.ssl_verify if connector else True)

pyclowder/api/v2/metadata.py

Whitespace-only changes.

pyclowder/client.py

Lines changed: 1 addition & 211 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,10 @@ def __init__(self, **kwargs):
2929
3030
:param ClowderClient client: Optional clowderclient to copy all parameters from, any additional parameters
3131
will override the values in this client
32-
:param string host: The root host url of the specific geostreaming API we are connecting to.
32+
:param string host: The root host url of the specific API we are connecting to.
3333
:param string key: The API key used to write to the API. Set this or `username`/`password` below.
3434
:param string username: HTTP Basic Authentication username. Set this or `key`.
3535
:param string password: HTTP Basic Authentication password. Set this or `key`.
36-
:param int retries: Number of times to retry before giving up.
37-
:param float timeout: Number of seconds to try to connect, and wait between retries.
38-
:param boolean ssl: Should ssl certificates be validated, default is true
3936
"""
4037

4138
# clone operator
@@ -45,17 +42,11 @@ def __init__(self, **kwargs):
4542
self.key = kwargs.get('key', client.key)
4643
self.username = kwargs.get('username', client.username)
4744
self.password = kwargs.get('password', client.password)
48-
self.retries = kwargs.get('retries', client.retries)
49-
self.timeout = kwargs.get('timeout', client.timeout)
50-
self.ssl = kwargs.get('ssl', client.ssl)
5145
else:
5246
self.host = kwargs.get('host', 'http://localhost:9000')
5347
self.key = kwargs.get('key', None)
5448
self.username = kwargs.get('username', None)
5549
self.password = kwargs.get('password', None)
56-
self.retries = kwargs.get('retries', 0)
57-
self.timeout = kwargs.get('timeout', 5)
58-
self.ssl = kwargs.get('ssl', True)
5950

6051
# make sure the host does not end with a slash
6152
self.host = self.host.rstrip('/')
@@ -65,204 +56,3 @@ def __init__(self, **kwargs):
6556
self.logger.warning("No key or username/password present.")
6657
if self.key and self.username and self.password:
6758
self.logger.info("Both key and username/password present, will use username/password for calls.")
68-
69-
def get(self, path, params=None, headers=None):
70-
"""
71-
Call HTTP GET against `path`. This version returns an object parsed from the response.
72-
73-
:param string path: Endpoint path relative to Clowder api.
74-
:param dict params: Additional parameters to pass to clowder.
75-
:param dict headers: Additional headers to pass to clowder, if not set content-type will be set.
76-
:return: the json-encoded content of a response.
77-
:raises: `requests.HTTPError`
78-
"""
79-
attempt = 0
80-
url = '%s/api/%s' % (self.host, path.lstrip('/'))
81-
if params is None:
82-
params = dict()
83-
if headers is None:
84-
headers = {'content-type': 'application/json'}
85-
if self.username and self.password:
86-
auth = (self.username, self.password)
87-
params['key'] = None
88-
elif self.key:
89-
auth = None
90-
params['key'] = self.key
91-
else:
92-
auth = None
93-
while True:
94-
try:
95-
response = requests.get(url, headers=headers, params=params,
96-
auth=auth, timeout=self.timeout, verify=self.ssl)
97-
response.raise_for_status()
98-
return response.json()
99-
except requests.HTTPError as e:
100-
attempt += 1
101-
if attempt > self.retries:
102-
self.logger.exception("Error calling GET url %s: %s" % (url, str(e)))
103-
raise e
104-
else:
105-
self.logger.debug("Error calling GET url %s: %s" % (url, str(e)))
106-
107-
def post(self, path, content, params=None, headers=None):
108-
"""
109-
Call HTTP POST against `path` with `content` in body.
110-
111-
:param path: Endpoint path relative to Clowder api.
112-
:param content: Content to send as the body of the request as a dict.
113-
:param dict params: Additional parameters to pass to clowder.
114-
:param dict headers: Additional headers to pass to clowder, if not set content-type will be set.
115-
:return: the json-encoded content of a response.
116-
:raises: `requests.HTTPError`
117-
"""
118-
attempt = 0
119-
url = '%s/api/%s' % (self.host, path.lstrip('/'))
120-
if params is None:
121-
params = dict()
122-
if headers is None:
123-
headers = {'content-type': 'application/json'}
124-
if self.username and self.password:
125-
auth = (self.username, self.password)
126-
params['key'] = None
127-
elif self.key:
128-
auth = None
129-
params['key'] = self.key
130-
else:
131-
auth = None
132-
while True:
133-
try:
134-
response = requests.post(url, data=json.dumps(content), headers=headers, params=params,
135-
auth=auth, timeout=self.timeout, verify=self.ssl)
136-
response.raise_for_status()
137-
return response.json()
138-
except requests.HTTPError as e:
139-
attempt += 1
140-
if attempt > self.retries:
141-
self.logger.exception("Error calling POST url %s: %s" % (url, str(e)))
142-
raise e
143-
else:
144-
self.logger.debug("Error calling POST url %s: %s" % (url, str(e)))
145-
146-
def delete(self, path, params=None, headers=None):
147-
"""
148-
Call HTTP DELETE against `path`.
149-
150-
:param path: Endpoint path relative to Clowder api.
151-
:param dict params: Additional parameters to pass to clowder.
152-
:param dict headers: Additional headers to pass to clowder
153-
:raises: `requests.HTTPError`
154-
"""
155-
attempt = 0
156-
url = '%s/api/%s' % (self.host, path.lstrip('/'))
157-
if params is None:
158-
params = dict()
159-
if headers is None:
160-
headers = {'content-type': 'application/json'}
161-
if self.username and self.password:
162-
auth = (self.username, self.password)
163-
params['key'] = None
164-
elif self.key:
165-
auth = None
166-
params['key'] = self.key
167-
else:
168-
auth = None
169-
while True:
170-
try:
171-
response = requests.delete(url, headers=headers, params=params,
172-
auth=auth, timeout=self.timeout, verify=self.ssl)
173-
response.raise_for_status()
174-
return response.json()
175-
except requests.HTTPError as e:
176-
attempt += 1
177-
if attempt > self.retries:
178-
self.logger.exception("Error calling DELETE url %s: %s" % (url, str(e)))
179-
raise e
180-
else:
181-
self.logger.debug("Error calling DELETE url %s: %s" % (url, str(e)))
182-
183-
def get_file(self, path, filename=None, params=None, headers=None):
184-
"""
185-
Call HTTP GET against `path` and writes the result to a file.
186-
187-
:param path: Endpoint path relative to Clowder api.
188-
:param filename: The name of the file, if not set a temporary file is created.
189-
:param dict params: Additional parameters to pass to clowder.
190-
:param dict headers: Additional headers to pass to clowder.
191-
:return: the filename where the output is written.
192-
:raises: `requests.HTTPError`
193-
"""
194-
attempt = 0
195-
url = '%s/api/%s' % (self.host, path.lstrip('/'))
196-
if params is None:
197-
params = dict()
198-
if headers is None:
199-
headers = {'content-type': 'application/json'}
200-
if self.username and self.password:
201-
auth = (self.username, self.password)
202-
params['key'] = None
203-
elif self.key:
204-
auth = None
205-
params['key'] = self.key
206-
else:
207-
auth = None
208-
if filename is None:
209-
(fd, filename) = tempfile.mkstemp(".tmp", "clowder")
210-
os.close(fd)
211-
212-
while True:
213-
try:
214-
response = requests.get(url, stream=True, headers=headers, params=params,
215-
auth=auth, timeout=self.timeout, verify=self.ssl)
216-
response.raise_for_status()
217-
with open(filename, mode="wb") as outputfile:
218-
for chunk in response.iter_content(chunk_size=10 * 1024):
219-
outputfile.write(chunk)
220-
return filename
221-
except requests.HTTPError as e:
222-
os.remove(filename)
223-
attempt += 1
224-
if attempt > self.retries:
225-
self.logger.exception("Error calling DELETE url %s: %s" % (url, e.response.text))
226-
raise e
227-
else:
228-
self.logger.debug("Error calling DELETE url %s: %s" % (url, e.response.text))
229-
except Exception:
230-
os.remove(filename)
231-
raise
232-
233-
def post_file(self, path, filename, params=None, headers=None):
234-
"""
235-
Call HTTP POST against `path` with `content` in body. Header with content-type is not required.
236-
237-
:param path: Endpoint path relative to Clowder api.
238-
:param filename: The name of the file to post.
239-
:param dict params: Additional parameters to pass to clowder.
240-
:param dict headers: Additional headers to pass to clowder.
241-
:return: the json-encoded content of a response.
242-
:raises: `requests.HTTPError`
243-
"""
244-
attempt = 0
245-
url = '%s/api/%s' % (self.host, path.lstrip('/'))
246-
if params is None:
247-
params = dict()
248-
if self.username and self.password:
249-
auth = (self.username, self.password)
250-
params['key'] = None
251-
elif self.key:
252-
auth = None
253-
params['key'] = self.key
254-
else:
255-
auth = None
256-
while True:
257-
try:
258-
response = requests.post(url, files={"File": open(filename, 'rb')}, headers=headers, params=params,
259-
auth=auth, timeout=self.timeout, verify=self.ssl)
260-
response.raise_for_status()
261-
return response.json()
262-
except requests.HTTPError as e:
263-
attempt += 1
264-
if attempt > self.retries:
265-
self.logger.exception("Error calling POST url %s: %s" % (url, str(e)))
266-
raise e
267-
else:
268-
self.logger.debug("Error calling POST url %s: %s" % (url, str(e)))

0 commit comments

Comments
 (0)