K-67: Validate if train/serve file is executable

Pipfile

@@ -1,6 +1,3 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
@@ -15,4 +12,4 @@ kaos = {editable = true, path = "./cli"}
 python_version = "3.7"
 
 [pipenv]
-allow_prereleases = true
+allow_prereleases = true

backend/kaos_backend/controllers/tests/__init__.py

@@ -39,8 +39,12 @@ def check_pipeline_exists_mock(pipeline_name):
 
 
 def create_test_file(dirname, filename):
-    f = open(os.path.join(dirname, filename), "w")
-    f.write("this is fake")
+    if filename in ['train', 'serve']:
+        f = open(os.path.join(dirname, filename), "w")
+        f.write("#!")
+    else:
+        f = open(os.path.join(dirname, filename), "w")
+        f.write("this is fake")
     f.close()
     return f
 

backend/kaos_backend/util/tests/test_validators.py

@@ -17,6 +17,14 @@ def create_file(path):
     return temp
 
 
+def make_executable_file(path):
+    with open(path, 'w') as executable_file:
+        line = "#!"
+        executable_file.write(line)
+    executable_file.close()
+    return executable_file
+
+
 def remove_el(l, el):
     ll = l[:]
     ll.remove(el)
@@ -52,15 +60,15 @@ def test_is_not_empty():
 def test_validate_bundle_structure_is_empty():
     with pytest.raises(InvalidBundleError, match="Bundle must be non-empty"):
         with TemporaryDirectory() as temp_dir:
-            BundleValidator.validate_bundle_structure(temp_dir, [])
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=None)
 
 
 def test_validate_bundle_structure_missing_root_directory():
     with pytest.raises(InvalidBundleError, match="Missing root directory in source-code bundle"):
         with TemporaryDirectory() as temp_dir:
             # temp file to avoid empty file exception
             temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True)
-            BundleValidator.validate_bundle_structure(temp_dir, [])
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=None)
             temp.close()
 
 
@@ -72,7 +80,7 @@ def test_validate_bundle_structure_too_many_directories():
             tempfile.mkdtemp(dir=temp_dir)
             tempfile.mkdtemp(dir=temp_dir)
 
-            BundleValidator.validate_bundle_structure(temp_dir, [])
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=None)
             temp.close()
 
 
@@ -83,7 +91,7 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory():
             temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True)
             tempfile.mkdtemp(dir=temp_dir)
 
-            BundleValidator.validate_bundle_structure(temp_dir, [])
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=None)
             temp.close()
 
 
@@ -93,7 +101,45 @@ def test_validate_bundle_structure_missing_model_directory():
             base_dir = tempfile.mkdtemp(dir=temp_dir)
             filename = os.path.join(base_dir, "Dockerfile")
             create_file(filename)
-            BundleValidator.validate_bundle_structure(temp_dir, [])
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=None)
+
+
+def test_validate_train_bundle_missing_executable_file():
+    with pytest.raises(InvalidBundleError,
+                       match="The train file cannot be executed. "
+                             "Please ensure that first line begins with the shebang '#!' to make it an executable"):
+        with TemporaryDirectory() as temp_dir:
+            base_dir = tempfile.mkdtemp(dir=temp_dir)
+            dockerfile = os.path.join(base_dir, "Dockerfile")
+            create_file(dockerfile)
+            model_dir = os.path.join(base_dir, "model")
+            os.mkdir(model_dir)
+            mode = "train"
+            for f in REQ_TRAINING_FILES:
+                create_file(os.path.join(model_dir, f))
+
+            train_file = os.path.join(model_dir, mode)
+            create_file(train_file)
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode)
+
+
+def test_validate_serve_bundle_missing_executable_file():
+    with pytest.raises(InvalidBundleError,
+                       match="The serve file cannot be executed. "
+                             "Please ensure that first line begins with the shebang '#!' to make it an executable"):
+        with TemporaryDirectory() as temp_dir:
+            base_dir = tempfile.mkdtemp(dir=temp_dir)
+            dockerfile = os.path.join(base_dir, "Dockerfile")
+            create_file(dockerfile)
+            model_dir = os.path.join(base_dir, "model")
+            os.mkdir(model_dir)
+            mode = "serve"
+            for f in REQ_INFERENCE_FILES:
+                create_file(os.path.join(model_dir, f))
+
+            serve_file = os.path.join(model_dir, mode)
+            create_file(serve_file)
+            BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode)
 
 
 @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases)

backend/kaos_backend/util/validators.py

@@ -21,6 +21,7 @@
 
 
 class BundleValidator:
+
     REQUIRED_INFERENCE_FILES = ["__init__.py",
                                 "serve",
                                 "web-requirements.txt"]
@@ -31,6 +32,8 @@ class BundleValidator:
 
     MODEL = "model"
 
+    SHEBANG = "#!"
+
     @classmethod
     def is_empty(cls, directory: str) -> bool:
         return all(len(files) == 0 for root, _, files in os.walk(directory))
@@ -63,7 +66,7 @@ def validate_file(cls, f, files):
             raise InvalidBundleError(f"Missing file {f} in model directory of source-code bundle")
 
     @classmethod
-    def validate_bundle_structure(cls, directory, req_files):
+    def validate_bundle_structure(cls, directory, req_files, mode):
         cls.validate_empty(directory)
         bundle_root, model_dir = None, None
         for root, dirs, files in os.walk(directory):
@@ -82,19 +85,41 @@ def validate_bundle_structure(cls, directory, req_files):
                     for f in req_files:
                         cls.validate_file(f, files)
 
+                    if mode:
+                        for file in files:
+                            if file == mode:
+                                file_path = os.path.join(model_dir, mode)
+                                cls.validate_is_file_executable(file_path, mode)
+
+    @classmethod
+    def validate_is_file_executable(cls, executable_file, mode):
+        f = open(executable_file)
+        first_line = f.readline()
+        if cls.SHEBANG not in first_line:
+            raise InvalidBundleError(f"The {mode} file cannot be executed. "
+                                     f"Please ensure that first line begins with the shebang '#!' "
+                                     f"to make it an executable")
+
     @classmethod
     def validate_inference_bundle_structure(cls, directory: str):
         req_files = cls.REQUIRED_INFERENCE_FILES
-        cls.validate_bundle_structure(directory, req_files)
+        mode = 'serve'
+        cls.validate_bundle_structure(directory, req_files, mode=mode)
 
     @classmethod
     def validate_notebook_bundle_structure(cls, directory):
-        cls.validate_bundle_structure(directory, [])
+        cls.validate_bundle_structure(directory, [], mode=None)
 
     @classmethod
     def validate_train_bundle_structure(cls, directory):
         req_files = cls.REQUIRED_TRAINING_FILES
-        cls.validate_bundle_structure(directory, req_files)
+        mode = 'train'
+        cls.validate_bundle_structure(directory, req_files, mode=mode)
+
+    @classmethod
+    def validate_source_bundle_structure(cls, directory):
+        req_files = cls.REQUIRED_TRAINING_FILES
+        cls.validate_bundle_structure(directory, req_files, mode=None)
 
 
 def validate_cpu_request(cpu):