Add --resolve-links to resolve remote links

Author: jochenklar

isimip_publisher/commands.py

@@ -154,8 +154,13 @@ def count_public_links():
 
 
 def fetch_files():
-    remote_files = files.list_files(settings.REMOTE_PATH, settings.PATH,
-                                    remote_dest=settings.REMOTE_DEST, suffix=settings.PATTERN['suffix'])
+    if settings.RESOLVE_LINKS:
+        remote_files = files.list_all(settings.REMOTE_PATH, settings.PATH,
+                                      remote_dest=settings.REMOTE_DEST, suffix=settings.PATTERN['suffix'])
+    else:
+        remote_files = files.list_files(settings.REMOTE_PATH, settings.PATH,
+                                        remote_dest=settings.REMOTE_DEST, suffix=settings.PATTERN['suffix'])
+
     datasets = patterns.match_datasets(settings.PATTERN, settings.REMOTE_PATH, remote_files,
                                        include=settings.INCLUDE, exclude=settings.EXCLUDE)
     validation.validate_datasets(settings.SCHEMA, settings.PATH, datasets)

isimip_publisher/main.py

@@ -97,6 +97,8 @@ def get_parser(add_path=False, add_subparsers=False):
                          help='Skip the registration of the DOI when inserting/updating a resource')
     parser.add_argument('--skip-checksum', dest='skip_checksum', action='store_true', default=False,
                          help='Skip the computation of the checksum when checking')
+    parser.add_argument('--resolve-links', dest='resolve_links', action='store_true', default=False,
+                         help='Resolve remote links as if they were files')
     parser.add_argument('--log-level', dest='log_level', default='WARN',
                         help='Log level (ERROR, WARN, INFO, or DEBUG)')
     parser.add_argument('--log-file', dest='log_file',

isimip_publisher/utils/files.py

@@ -52,6 +52,10 @@ def list_links(base_path, path, remote_dest=None, suffix=None):
     return list_files(base_path, path, remote_dest=remote_dest, suffix=suffix, find_type='l')
 
 
+def list_all(base_path, path, remote_dest=None, suffix=None):
+    return list_files(base_path, path, remote_dest=remote_dest, suffix=suffix, find_type=None)
+
+
 def filter_links(public_path, target_path, path, links):
     filtered_links = []
     for link_path in links: