Prevent Index creation during import (mlcommons#205)

Author: sujik18

mlc/action.py

@@ -165,6 +165,10 @@ def load_repos(self):
             logger.error(f"Error reading file: {e}")
             return None
 
+    def get_index(self):
+        if self._index is None:
+            self._index = Index(self.repos_path, self.repos)
+        return self._index
 
     def __init__(self):        
         setup_logging(log_path=os.getcwd(), log_file='.mlc-log.txt')
@@ -200,7 +204,7 @@ def __init__(self):
 
         self.repos = self.load_repos_and_meta()
         #logger.info(f"In Action class: {self.repos_path}")
-        self.index = Index(self.repos_path, self.repos)
+        self._index = None
 
 
     def add(self, i):
@@ -380,7 +384,7 @@ def rm(self, i):
 
                 logger.info(f"{target_name} item: {item_path} has been successfully removed")
 
-            self.index.rm(item_meta, target_name, item_path)
+            self.get_index().rm(item_meta, target_name, item_path)
 
         return {
             "return": 0,
@@ -413,7 +417,7 @@ def save_new_meta(self, i, item_id, item_name, target_name, item_path, repo):
         if save_result["return"] > 0:
             return save_result
 
-        self.index.add(item_meta, target_name, item_path, repo)
+        self.get_index().add(item_meta, target_name, item_path, repo)
         return {'return': 0}
 
     def update(self, i):
@@ -482,7 +486,7 @@ def update(self, i):
             # Save the updated meta back to the item
             item.meta = meta
             save_result = utils.save_json(item_meta_path, meta=meta)
-            self.index.update(meta, target_name, item.path, item.repo)
+            self.get_index().update(meta, target_name, item.path, item.repo)
 
         return {'return': 0, 'message': f"Tags updated successfully for {len(found_items)} item(s).", 'list': found_items }
 
@@ -641,13 +645,13 @@ def mv(self, run_args):
         #Put the src uid to the destination path
         dest.meta['uid'] = src.meta['uid']
         dest._save_meta()
-        self.index.update(dest.meta, target_name, dest.path, dest.repo)
+        self.get_index().update(dest.meta, target_name, dest.path, dest.repo)
         logger.info(f"""Item with uid {dest.meta['uid']} successfully moved from {src.path} to {dest.path}""")
 
         return {'return': 0, 'src': src, 'dest': dest}
 
     def search(self, i):
-        indices = self.index.indices
+        indices = self.get_index().indices
         target = i.get('target_name', self.action_type)
         target_index = indices.get(target)
         result = []

mlc/index.py

@@ -131,13 +131,10 @@ def rm(self, meta, folder_type, path):
         self._save_indices()
 
     def get_item_mtime(self,file):
-        # logger.debug(f"Getting latest modified time for file: {file}")
         latest = 0
         t = os.path.getmtime(file)
         if t > latest:
             latest = t
-            logger.debug(f"Latest modified time updated to: {latest}")
-        # logger.debug("No changes in modified time detected.")
         return latest
 
     def build_index(self):
@@ -203,7 +200,7 @@ def build_index(self):
             repo_path = repo.path #os.path.join(self.repos_path, repo)
             if not os.path.isdir(repo_path):
                 continue
-            logger.debug(f"Checking repository: {repo_path}")
+            logger.debug(f"------------Checking repository: {repo_path}---------------")
             # Filter for relevant directories in the repo
             for folder_type in ["script", "cache", "experiment"]:
                 logger.debug(f"Checking folder type: {folder_type}")
@@ -230,12 +227,15 @@ def build_index(self):
                         config_path = json_path
                     else:
                         logger.debug(f"No config file found in {automation_path}, skipping")
+                        delete_flag = False
                         if automation_dir in self.modified_times:
                             del self.modified_times[automation_dir]
                         if any(automation_dir in item["path"] for item in self.indices[folder_type]):
                             logger.debug(f"Removed index entry (if it exists) for {folder_type} : {automation_dir}")
+                            delete_flag = True
                             self._remove_index_entry(automation_path)
-                        self._save_indices()
+                        if delete_flag:
+                            self._save_indices()
                         continue
                     current_item_keys.add(config_path)
                     mtime = self.get_item_mtime(config_path)

mlc/logger.py

@@ -9,6 +9,7 @@ class ColoredFormatter(logging.Formatter):
     """Custom formatter class to add colors to log levels"""
     COLORS = {
         'INFO': Fore.GREEN,
+        'DEBUG': Fore.CYAN + Style.DIM,
         'WARNING': Fore.YELLOW,
         'ERROR': Fore.RED
     }

mlc/main.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 import inspect
 import shlex
+import unicodedata
 from . import utils
 
 from .action import Action, default_parent
@@ -43,7 +44,7 @@ def _load_meta(self):
             logger.info(f"No meta file found in {self.path}")
 
     def search(self, i):
-        indices = self.action_object.index.indices
+        indices = self.action_object.get_index().indices
         target_index = indices.get(self.automation_type)
         result = []
         if target_index:
@@ -178,7 +179,8 @@ def configure_logging(args):
         args.extra[:] = [log_flag_aliases.get(a, a) for a in args.extra]
         for flag, level in log_levels.items():
             if flag in args.extra:
-                logger.setLevel(level)
+                if not logger.isEnabledFor(level):
+                    logger.setLevel(level)
                 args.extra.remove(flag)
 
 
@@ -219,6 +221,7 @@ def is_quoted(arg):
     return (arg.startswith("'") and arg.endswith("'")) or \
            (arg.startswith('"') and arg.endswith('"'))
 
+
 def check_raw_arguments_for_non_ascii():
     bad_args = []