Skip to content

Commit aa57e01

Browse files
yuvipandayuvipanda
authored
Merge pull request #1315 from manics/zenodo
Get Zenodo working again
2 parents de496f8 + fc7de27 commit aa57e01

File tree

4 files changed

+101
-25
lines changed

4 files changed

+101
-25
lines changed

‎repo2docker/contentproviders/doi.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,15 @@ def doi2url(self, doi):
5151
try:
5252
resp = self._request(f"https://doi.org/{doi}")
5353
resp.raise_for_status()
54-
# If the DOI doesn't resolve, just return URL
55-
except HTTPError:
56-
return doi
54+
except HTTPError as e:
55+
# If the DOI doesn't exist, just return URL
56+
if e.response.status_code == 404:
57+
return doi
58+
# Reraise any other errors because if the DOI service is down (or
59+
# we hit a rate limit) we don't want to silently continue to the
60+
# default Git provider as this leads to a misleading error.
61+
logging.error(f"DOI {doi} does not resolve: {e}")
62+
raise
5763
return resp.url
5864
else:
5965
# Just return what is actulally just a URL

‎repo2docker/contentproviders/zenodo.py

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,26 @@ def __init__(self):
2222
"hostname": [
2323
"https://sandbox.zenodo.org/record/",
2424
"http://sandbox.zenodo.org/record/",
25+
"http://sandbox.zenodo.org/records/",
2526
],
2627
"api": "https://sandbox.zenodo.org/api/records/",
27-
"filepath": "files",
28-
"filename": "filename",
29-
"download": "links.download",
28+
"files": "links.files",
29+
"filepath": "entries",
30+
"filename": "key",
31+
"download": "links.content",
3032
"type": "metadata.upload_type",
3133
},
3234
{
33-
"hostname": ["https://zenodo.org/record/", "http://zenodo.org/record/"],
35+
"hostname": [
36+
"https://zenodo.org/record/",
37+
"http://zenodo.org/record/",
38+
"https://zenodo.org/records/",
39+
],
3440
"api": "https://zenodo.org/api/records/",
35-
"filepath": "files",
36-
"filename": "filename",
37-
"download": "links.download",
41+
"files": "links.files",
42+
"filepath": "entries",
43+
"filename": "key",
44+
"download": "links.content",
3845
"type": "metadata.upload_type",
3946
},
4047
{
@@ -43,6 +50,7 @@ def __init__(self):
4350
"http://data.caltech.edu/records/",
4451
],
4552
"api": "https://data.caltech.edu/api/record/",
53+
"files": "",
4654
"filepath": "metadata.electronic_location_and_access",
4755
"filename": "electronic_name.0",
4856
"download": "uniform_resource_identifier",
@@ -69,9 +77,17 @@ def fetch(self, spec, output_dir, yield_output=False):
6977
f'{host["api"]}{record_id}',
7078
headers={"accept": "application/json"},
7179
)
72-
7380
record = resp.json()
7481

82+
if host["files"]:
83+
yield f"Fetching Zenodo record {record_id} files.\n"
84+
files_url = deep_get(record, host["files"])
85+
resp = self.urlopen(
86+
files_url,
87+
headers={"accept": "application/json"},
88+
)
89+
record = resp.json()
90+
7591
files = deep_get(record, host["filepath"])
7692
only_one_file = len(files) == 1
7793
for file_ref in files:

‎tests/unit/contentproviders/test_doi.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,14 @@ def test_url_headers(requests_mock):
3030
assert result.request.headers["User-Agent"] == f"repo2docker {__version__}"
3131

3232

33-
def test_unresolving_doi():
33+
@pytest.mark.parametrize(
34+
"requested_doi, expected",
35+
[
36+
("10.5281/zenodo.3242074", "https://zenodo.org/records/3242074"),
37+
# Unresolving DOI:
38+
("10.1/1234", "10.1/1234"),
39+
],
40+
)
41+
def test_doi2url(requested_doi, expected):
3442
doi = DoiProvider()
35-
36-
fakedoi = "10.1/1234"
37-
assert doi.doi2url(fakedoi) is fakedoi
43+
assert doi.doi2url(requested_doi) == expected

‎tests/unit/contentproviders/test_zenodo.py

Lines changed: 58 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,31 @@ def test_fetch_software_from_github_archive(requests_mock):
9292
# we "fetch" a local ZIP file to simulate a Zenodo record created from a
9393
# GitHub repository via the Zenodo-GitHub integration
9494
with zenodo_archive() as zen_path:
95-
mock_response = {
95+
mock_record = {
9696
"files": [
9797
{
9898
"filename": "some_dir/afake.zip",
99-
"links": {"download": f"file://{zen_path}"},
10099
}
101100
],
101+
"links": {
102+
"files": "https://zenodo.org/api/records/1234/files",
103+
},
102104
"metadata": {"upload_type": "other"},
103105
}
104-
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
106+
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)
107+
108+
mock_record_files = {
109+
"entries": [
110+
{
111+
"key": "some_dir/afake.zip",
112+
"links": {"content": f"file://{zen_path}"},
113+
}
114+
],
115+
}
116+
requests_mock.get(
117+
"https://zenodo.org/api/records/1234/files", json=mock_record_files
118+
)
119+
105120
requests_mock.get(f"file://{zen_path}", content=open(zen_path, "rb").read())
106121

107122
zen = Zenodo()
@@ -121,18 +136,33 @@ def test_fetch_software(requests_mock):
121136
# we "fetch" a local ZIP file to simulate a Zenodo software record with a
122137
# ZIP file in it
123138
with zenodo_archive() as zen_path:
124-
mock_response = {
139+
mock_record = {
125140
"files": [
126141
{
127142
# this is the difference to the GitHub generated one,
128143
# the ZIP file isn't in a directory
129144
"filename": "afake.zip",
130-
"links": {"download": f"file://{zen_path}"},
131145
}
132146
],
147+
"links": {
148+
"files": "https://zenodo.org/api/records/1234/files",
149+
},
133150
"metadata": {"upload_type": "software"},
134151
}
135-
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
152+
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)
153+
154+
mock_record_files = {
155+
"entries": [
156+
{
157+
"key": "afake.zip",
158+
"links": {"content": f"file://{zen_path}"},
159+
}
160+
],
161+
}
162+
requests_mock.get(
163+
"https://zenodo.org/api/records/1234/files", json=mock_record_files
164+
)
165+
136166
requests_mock.get(f"file://{zen_path}", content=open(zen_path, "rb").read())
137167

138168
with TemporaryDirectory() as d:
@@ -151,20 +181,38 @@ def test_fetch_data(requests_mock):
151181
# we "fetch" a local ZIP file to simulate a Zenodo data record
152182
with zenodo_archive() as a_zen_path:
153183
with zenodo_archive() as b_zen_path:
154-
mock_response = {
184+
mock_record = {
155185
"files": [
156186
{
157187
"filename": "afake.zip",
158-
"links": {"download": f"file://{a_zen_path}"},
159188
},
160189
{
161190
"filename": "bfake.zip",
162-
"links": {"download": f"file://{b_zen_path}"},
163191
},
164192
],
193+
"links": {
194+
"files": "https://zenodo.org/api/records/1234/files",
195+
},
165196
"metadata": {"upload_type": "data"},
166197
}
167-
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
198+
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)
199+
200+
mock_record_files = {
201+
"entries": [
202+
{
203+
"key": "afake.zip",
204+
"links": {"content": f"file://{a_zen_path}"},
205+
},
206+
{
207+
"key": "bfake.zip",
208+
"links": {"content": f"file://{b_zen_path}"},
209+
},
210+
],
211+
}
212+
requests_mock.get(
213+
"https://zenodo.org/api/records/1234/files", json=mock_record_files
214+
)
215+
168216
requests_mock.get(
169217
f"file://{a_zen_path}", content=open(a_zen_path, "rb").read()
170218
)

0 commit comments

Comments
 (0)