Add option to delete files and fix uploading of file that already exists

Author: tmorrell

caltechdata_api/caltechdata_edit.py

@@ -81,33 +81,44 @@ def caltechdata_edit(
     for idv in ids:
         metadata["id"] = idv
 
-        if files:
-            # Files to delete
+        if files or delete:
+            # Files to add or delete
             fjson = {}
             c = session()
-            existing = c.get(api_url + idv)
-            file_info = existing.json()["metadata"]
+            existing_meta = c.get(api_url + idv)
+            file_info = existing_meta.json()["metadata"]
+            existing = {}
+            if "electronic_location_and_access" in file_info:
+                for ex in file_info["electronic_location_and_access"]:
+                    name = ex["electronic_name"][0]
+                    fu = ex["uniform_resource_identifier"].split("/")[-2]
+                    existing[name] = fu
+
+            # File ids to delete
             fids = []
-            for f in files:  # Check if new files match existing
-                if "electronic_location_and_access" in file_info:
-                    for ex in file_info["electronic_location_and_access"]:
-                        name = ex["electronic_name"][0]
-                        fu = ex["uniform_resource_identifier"].split("/")[-2]
-                        if name == f:
+
+            if files:
+                for f in files:  # Check if new files match existing
+                    # In case we have a path
+                    fname = f.split("/")[-1]
+                    if fname in existing:
+                        fids.append(existing[fname])
+                # Now upload new files
+                fileinfo = [send_s3(f, token, production) for f in files]
+                fjson["new"] = fileinfo
+
+            if delete:
+                for d in delete:
+                    if d in existing:
+                        fids.append(existing[d])
+                    # Also check if we are deleting by extenson
+                    for e in existing.keys():
+                        if e.split(".")[-1] == d:
                             fids.append(fu)
-                        for d in delete:
-                            if name == d:
-                                fids.append(fu)
-                            if name.split(".")[-1] == d:
-                                fids.append(fu)
+
             if len(fids) > 0:
                 fjson = {"delete": fids}
 
-            # upload new
-            print(files)
-            fileinfo = [send_s3(f, token, production) for f in files]
-
-            fjson["new"] = fileinfo
             metadata["files"] = fjson
 
         dat = json.dumps({"record": metadata})

edit.py

@@ -4,20 +4,24 @@
 parser = argparse.ArgumentParser(description=\
         "Write files and a DataCite 4 standard json record\
         to CaltechDATA repository")
-parser.add_argument('json_file', nargs=1, help=\
+parser.add_argument('json_file', nargs='?', default=None, help=\
             'file name for json DataCite metadata file')
 parser.add_argument('-ids', nargs='*', help='CaltechDATA IDs')
 parser.add_argument('-fnames', nargs='*', help='New Files')
 parser.add_argument('-schema', default="40", help="Metadata Schema")
+parser.add_argument('-delete', nargs='*', default="{}", help="Filename or extension to delete")
 args = parser.parse_args()
 
 #Get access token from TIND set as environment variable with source token.bash
 token = os.environ['TINDTOK']
 
-metaf = open(args.json_file[0], 'r')
-metadata = json.load(metaf)
+if args.json_file:
+    metaf = open(args.json_file, 'r')
+    metadata = json.load(metaf)
+else:
+    metadata={}
 
 production = True
 
-response = caltechdata_edit(token, args.ids, metadata, args.fnames, {'pdf'}, production, args.schema)
+response = caltechdata_edit(token, args.ids, metadata, args.fnames, args.delete, production, args.schema)
 print(response)