Skip to content

Commit 4c8588b

Browse files
committed
minor updates to europeana fetch
- update --limit help text - use standard (for this repository) backup_factor=10 - use itemgettr for all sorts - remove redundant facet logging
1 parent 5b1aa20 commit 4c8588b

File tree

1 file changed

+3
-6
lines changed

1 file changed

+3
-6
lines changed

scripts/1-fetch/europeana_fetch.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def parse_arguments():
9595
"--limit",
9696
type=int,
9797
default=5,
98-
help="Limit number of providers for testing.",
98+
help="Limit number of data providers",
9999
)
100100
args = parser.parse_args()
101101
if not args.enable_save and args.enable_git:
@@ -106,7 +106,7 @@ def parse_arguments():
106106
def get_requests_session():
107107
"""Create a requests session with retry."""
108108
max_retries = Retry(
109-
total=5, backoff_factor=5, status_forcelist=shared.STATUS_FORCELIST
109+
total=5, backoff_factor=10, status_forcelist=shared.STATUS_FORCELIST
110110
)
111111
session = requests.Session()
112112
session.mount("https://", HTTPAdapter(max_retries=max_retries))
@@ -260,7 +260,7 @@ def get_facet_list(session, facet_field):
260260
)
261261

262262
# Sort by count descending
263-
all_values.sort(key=lambda x: x["count"], reverse=True)
263+
all_values.sort(key=itemgetter("count"), reverse=True)
264264
return all_values
265265

266266

@@ -439,9 +439,6 @@ def main():
439439
providers_full = get_facet_list(session, "DATA_PROVIDER")
440440
rights_full = get_facet_list(session, "RIGHTS")
441441

442-
LOGGER.info(f"Facet providers loaded: {len(providers_full)}")
443-
LOGGER.info(f"Facet rights loaded: {len(rights_full)}")
444-
445442
# Pass facets to fetch functions
446443
data_no_theme = fetch_europeana_data_without_themes(
447444
session, providers_full, rights_full, limit=args.limit

0 commit comments

Comments
 (0)