run api as group, clean up file transfer (#88)

Author: jacobfilik

Charts/web-conexs/templates/api_deployment.yaml

@@ -15,6 +15,8 @@ spec:
     spec:
       securityContext:
         runAsUser: 1000
+        supplementalGroups:
+          - {{.Values.security.securityContext.runAsGroup}}
       volumes:
         - name: data-pv-volume
           persistentVolumeClaim:

web-conexs-api/src/slurm_submission_service/filetransfer.py

@@ -2,7 +2,6 @@
 import logging
 import os
 import shutil
-import tempfile
 from pathlib import Path
 
 logger = logging.getLogger(__name__)
@@ -13,11 +12,6 @@ def check_filesystem(path):
     Path(path).stat()
 
 
-def make_directory(path):
-    logger.info(f"Attempting to make directory {path}")
-    Path(path).mkdir(parents=True)
-
-
 def copy_directory(source, destination):
     logger.info(f"Attempting to make copy {source} to {destination}")
 
@@ -35,14 +29,19 @@ def copy_multiple_files(abs_paths: list[str], destination):
         shutil.copy(p, destination)
 
 
-def transfer_inputs(file_map: dict[str, str], final_dir):
-    with tempfile.TemporaryDirectory() as tmpdirname:
-        for k, v in file_map.items():
-            input_file = tmpdirname + "/" + k
-            with open(input_file, "w+") as f:
-                f.write(v)
+def transfer_inputs(file_map: dict[str, str], destination):
+    dest = Path(destination)
+    parent = dest.parent
 
-        return copy_directory(tmpdirname, final_dir)
+    if not parent.exists:
+        parent.mkdir()
+
+    dest.mkdir()
+
+    for k, v in file_map.items():
+        input_file = dest / k
+        with open(input_file, "w+") as f:
+            f.write(v)
 
 
 def transfer_results(simulation_type_id, result_dir, storage_dir):
@@ -72,17 +71,16 @@ def transfer_results(simulation_type_id, result_dir, storage_dir):
     parent = store_path.parent
 
     if not parent.exists:
-        parent.mkdir(mode=0o755)
+        parent.mkdir()
 
     if not store_path.exists:
-        store_path.mkdir(mode=0o755)
+        store_path.mkdir()
 
     shutil.copytree(
         result_dir,
         storage_dir,
         ignore=ignore_pattern,
         dirs_exist_ok=True,
-        copy_function=shutil.copyfile,
     )
 
 

web-conexs-api/src/slurm_submission_service/submitter.py

@@ -46,6 +46,8 @@ def run_update():
 
 
 def main():
+    os.umask(0o007)
+
     rootlogger = logging.getLogger()
     formatter = logging.Formatter(
         "%(asctime)s - %(levelname)s - %(name)s - %(message)s",

web-conexs-api/tests/test_filetransfer.py

@@ -1,11 +1,11 @@
+import os
 from pathlib import Path
 
 from slurm_submission_service.filetransfer import (
     check_filesystem,
     clean_up,
     copy_directory,
     copy_multiple_files,
-    make_directory,
     transfer_inputs,
     transfer_results,
 )
@@ -18,13 +18,6 @@ def test_check_filesystem(tmp_path: Path):
     assert True
 
 
-def test_make_directory(tmp_path: Path):
-    d = tmp_path / "test_dir"
-    assert not d.exists()
-    make_directory(str(d))
-    assert d.exists()
-
-
 def test_copy_dir_file(tmp_path: Path):
     job_text = "job"
     filename = "job.inp"
@@ -48,6 +41,7 @@ def test_copy_dir_file(tmp_path: Path):
 
 
 def test_transfer_inputs(tmp_path: Path):
+    os.umask(0o007)
     job_text = "job"
     filename = "job.inp"
     file_map = {}
@@ -58,28 +52,39 @@ def test_transfer_inputs(tmp_path: Path):
 
     output = d / filename
 
+    mode = oct(os.stat(d).st_mode)
+
+    assert mode[-3:] == "770"
+
     assert output.exists()
 
     assert output.read_text() == job_text
 
 
 def test_transfer_results(tmp_path: Path):
+    os.umask(0o007)
     job_text = "job"
     filename = "job.inp"
     bad_file = "test.gbw"
     s = tmp_path / "source"
     s.mkdir()
+    os.chmod(s, 0o770)
+
     p = s / filename
     p.write_text("job")
 
     b = s / bad_file
     b.touch()
 
     orca = tmp_path / "orca"
-    orca.mkdir()
 
     transfer_results(1, str(s), str(orca))
 
+    print(oct(os.stat(orca).st_mode))
+
+    mode = oct(os.stat(orca).st_mode)
+
+    assert mode[-3:] == "770"
     print([str(x) for x in s.iterdir()])
     print([str(x) for x in orca.iterdir()])