Merge pull request #6 from ral-facilities/revert-5-revert-4-3_search_files

kevinphippsstfc · web-flow · commit 63599c424cc0 · 2025-11-26T14:51:14.000Z
Revert "Revert "Add search_files.py, refactor common functionality into common.py""

Re-reverting this now that we are ready for this functionality to be on main
diff --git a/README.md b/README.md
@@ -68,3 +68,63 @@ options:
                         complete with an interval of this many minutes. Non-
                         positive values will disable monitoring.
 ```
+
+## search_files
+
+Performs DataGateway searches for Datafiles matching the provided query.
+```bash
+python3 search_files.py input-file.txt 'visitId:"AB1234-1"' --username=abc12345
+```
+
+Full help text and description of arguments is available with the `--help` command:
+```bash
+python3 search_files.py --help
+```
+```
+usage: search_files [-h] [--url URL] [-a AUTHENTICATOR] -u USERNAME
+                    [-p PASSWORD_FILE] [-m MAX_RESULTS]
+                    output_file query
+
+Performs DataGateway searches for Datafiles matching the provided query. These paths will be written to file in
+batches, and can then be inspected and filtered further if needed before submitting using queue_file_downloads.
+
+positional arguments:
+  output_file           File to append newline separated paths to.
+                        This can then be provided as an input to queue_file_downloads.
+  query                 Lucene syntax formatted search query. Full help text and examples can be found
+                        in the DataGateway UI. Note that wildcards can significantly increase the time
+                        taken to perform a search, and the more specific the search query is the more
+                        efficient it will be. Some example searches are:
+                            'visitId:AB1234'
+                                Search for all Datafiles in all parts of proposal
+                            'visitId:"AB1234-1"'
+                                Search for all Datafiles in a (part) visit
+                            'location.fileName:"config.txt"'
+                                Search for Datafiles with a specific file name and extension (both required)
+                            'location.fileName:config'
+                                Search for Datafiles with a specific name but any extension
+                            'location.fileName:txt'
+                                Search for Datafiles with the extension 'txt', but no requirement on the name
+                            'location:raw'
+                                Search for Datafiles with the directory 'raw' somewhere in their path
+                            'location:(raw processed)'
+                                Search for Datafiles with either of two directories somewhere in their path
+                            'location.exact:/dls/i0/data/2000'
+                                Search for Datafiles in any subdirectory of the provided path (case sensitive)
+                            'location.exact:/dls/i0/data/202?/*/raw/config.txt'
+                                Search for a full path with wildcards (case sensitive)
+                            '+location.exact:/dls/i0/data/202? +location:(raw processed) +location.fileName:txt'
+                                Search for multiple criteria (all of which are required to match)
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --url URL             The url address of the DataGateway instance to submit requests to.
+  -a AUTHENTICATOR, --authenticator AUTHENTICATOR
+                        The authentication mechanism to use for DataGateway login.
+  -u USERNAME, --username USERNAME
+                        The username used for DataGateway login.
+  -p PASSWORD_FILE, --password-file PASSWORD_FILE
+                        Location of file containing password for DataGateway login. If not provided, the password will need to be provided by prompt.
+  -m MAX_RESULTS, --max-results MAX_RESULTS
+                        The maximum number of results to request in a single batch. If unset, the server default value will be used.
+```
diff --git a/common.py b/common.py
@@ -0,0 +1,88 @@
+from argparse import ArgumentParser
+from getpass import getpass
+import json
+from urllib.parse import quote
+
+import requests
+
+
+VERIFY = True  # If False then ssl certs won't be checked, useful for development
+
+
+def add_common_args(parser: ArgumentParser) -> None:
+    """
+    Add the common arguments url, authenticator, username and password-file to `parser`.
+
+    Args:
+        parser (ArgumentParser): ArgumentParser for the program.
+    """
+    parser.add_argument(
+        "--url",
+        type=str,
+        default="https://datagateway.diamond.ac.uk",
+        help="The url address of the DataGateway instance to submit requests to.",
+    )
+    parser.add_argument(
+        "-a",
+        "--authenticator",
+        type=str,
+        default="ldap",
+        help="The authentication mechanism to use for DataGateway login.",
+    )
+    parser.add_argument(
+        "-u",
+        "--username",
+        type=str,
+        required=True,
+        help="The username used for DataGateway login.",
+    )
+    parser.add_argument(
+        "-p",
+        "--password-file",
+        type=str,
+        help=(
+            "Location of file containing password for DataGateway login. If not "
+            "provided, the password will need to be provided by prompt."
+        ),
+    )
+
+
+def get_password(password_file: "str | None") -> str:
+    """
+    Load the user's password from `password_file` if provided, otherwise prompt for it.
+
+    Args:
+        password_file (str): Optional path of file containing the user's password.
+
+    Returns:
+        str: The user's password
+    """
+    if password_file is None:
+        return getpass()
+    else:
+        with open(password_file) as f:
+            return f.readline().strip()
+
+
+def login(base_url: str, authenticator: str, username: str, password: str) -> str:
+    """
+    Args:
+        base_url (str): URL for DataGateway without path.
+        authenticator (str): Authentication mechanism to use.
+        username (str): Username to use.
+        password (str): Password to use.
+
+    Raises:
+        RuntimeError: If a status code other than 200 is returned.
+
+    Returns:
+        str: ICAT session id.
+    """
+    url = f"{base_url}/topcat/user/session"
+    encoded_password = quote(json.dumps(password)[1:-1])
+    data = {"plugin": authenticator, "username": username, "password": encoded_password}
+    response = requests.post(url=url, data=data, verify=VERIFY)
+    if response.status_code != 200:
+        raise RuntimeError(response.text)
+
+    return json.loads(response.content)["sessionId"]
diff --git a/queue_file_downloads.py b/queue_file_downloads.py
@@ -2,34 +2,11 @@
  
 import argparse
 from datetime import datetime
-from getpass import getpass
 import json
 from time import sleep
 import requests
-from urllib.parse import quote
 
-def login(base_url: str, authenticator: str, username: str, password: str) -> str:
-    """
-    Args:
-        base_url (str): URL for DataGateway without path.
-        authenticator (str): Authentication mechanism to use.
-        username (str): Username to use.
-        password (str): Password to use.
-
-    Raises:
-        RuntimeError: If a status code other than 200 is returned.
-
-    Returns:
-        str: ICAT session id.
-    """
-    url = f"{base_url}/topcat/user/session"
-    encoded_password = quote(json.dumps(password)[1:-1])
-    data = {"plugin": authenticator, "username": username, "password": encoded_password}
-    response = requests.post(url=url, data=data)
-    if response.status_code != 200:
-        raise RuntimeError(response.text)
-
-    return json.loads(response.content)["sessionId"]
+from common import VERIFY, add_common_args, get_password, login
 
 
 def queue_all_files(
@@ -119,14 +96,15 @@ def queue_files(
     Returns:
         int: The Download id.
     """
+    url = f"{base_url}/topcat/user/queue/files"
     data = {
         "sessionId": session_id,
         "transport": transport,
         "fileName": file_name,
         "email": email,
         "files": files,
     }
-    response = requests.post(url=base_url + "/topcat/user/queue/files", data=data)
+    response = requests.post(url=url, data=data, verify=VERIFY)
     if response.status_code != 200:
         raise RuntimeError(response.text)
 
@@ -160,20 +138,21 @@ def monitor(
     """
     url = base_url + "/topcat/user/downloads/status"
     params = {"sessionId": session_id, "downloadIds": downloads}
-    response = requests.get(url=url, params=params)
+    response = requests.get(url=url, params=params, verify=VERIFY)
     if response.status_code != 200:
         raise RuntimeError(response.text)
     content = json.loads(response.content)
     print(content)
 
     while any([s in {"QUEUED", "PAUSED", "PREPARING", "RESTORING"} for s in content]):
+        sessions_url = f"{base_url}/datagateway-api/sessions"
         headers = {"Authorization": f"Bearer {session_id}"}
-        requests.put(url=base_url + "/datagateway-api/sessions", headers=headers)
+        requests.put(url=sessions_url, headers=headers, verify=VERIFY)
         if response.status_code != 200:
             raise RuntimeError(response.text)
 
         sleep(monitor_sleep * 60)
-        response = requests.get(url=url, params=params)
+        response = requests.get(url=url, params=params, verify=VERIFY)
         if response.status_code != 200:
             raise RuntimeError(response.text)
 
@@ -205,35 +184,7 @@ def monitor(
             "displayed in the DataGateway UI."
         ),
     )
-    parser.add_argument(
-        "--url",
-        type=str,
-        default="https://datagateway.diamond.ac.uk",
-        help="The url address of the DataGateway instance to submit requests to.",
-    )
-    parser.add_argument(
-        "-a",
-        "--authenticator",
-        type=str,
-        default="ldap",
-        help="The authentication mechanism to use for DataGateway login.",
-    )
-    parser.add_argument(
-        "-u",
-        "--username",
-        type=str,
-        required=True,
-        help="The username used for DataGateway login.",
-    )
-    parser.add_argument(
-        "-p",
-        "--password-file",
-        type=str,
-        help=(
-            "Location of file containing password for DataGateway login. If not "
-            "provided, the password will need to be provided by prompt."
-        ),
-    )
+    add_common_args(parser)
     parser.add_argument(
         "--download-name",
         type=str,
@@ -276,12 +227,7 @@ def monitor(
     )
     args = parser.parse_args()
 
-    if args.password_file is None:
-        password = getpass()
-    else:
-        with open(args.password_file) as f:
-            password = f.readline().strip()
-
+    password = get_password()
     session_id = login(
         base_url=args.url,
         authenticator=args.authenticator,
diff --git a/search_files.py b/search_files.py
