@@ -510,7 +510,7 @@ def parse_gen_plugin_config(
510510 sys .exit (1 )
511511
512512
513- def log_system_info (log_path : str , system_info : SystemInfo , logger : logging .Logger ):
513+ def log_system_info (log_path : str | None , system_info : SystemInfo , logger : logging .Logger ):
514514 """dump system info object to json log
515515
516516 Args:
@@ -544,54 +544,61 @@ def main(arg_input: Optional[list[str]] = None):
544544 config_reg = ConfigRegistry ()
545545 parser , plugin_subparser_map = build_parser (plugin_reg , config_reg )
546546
547- top_level_args , plugin_arg_map = process_args (arg_input , list (plugin_subparser_map .keys ()))
548-
549- parsed_args = parser .parse_args (top_level_args )
550-
551- if parsed_args .log_path and parsed_args .subcmd not in ["gen-plugin-config" , "describe" ]:
552- log_path = os .path .join (
553- parsed_args .log_path ,
554- f"scraper_logs_{ datetime .datetime .now ().strftime ('%Y_%m_%d-%I_%M_%S_%p' )} " ,
555- )
556- os .makedirs (log_path )
557- else :
558- log_path = None
559-
560- logger = setup_logger (parsed_args .log_level , log_path )
561- if log_path :
562- logger .info ("Log path: %s" , log_path )
563-
564- if parsed_args .subcmd == "describe" :
565- parse_describe (parsed_args , plugin_reg , config_reg , logger )
566-
567- if parsed_args .subcmd == "gen-plugin-config" :
568- parse_gen_plugin_config (parsed_args , plugin_reg , config_reg , logger )
569-
570- parsed_plugin_args = {}
571- for plugin , plugin_args in plugin_arg_map .items ():
572- try :
573- parsed_plugin_args [plugin ] = plugin_subparser_map [plugin ][0 ].parse_args (plugin_args )
574- except Exception :
575- logger .exception ("Exception parsing args for plugin: %s" , plugin )
547+ try :
548+ top_level_args , plugin_arg_map = process_args (arg_input , list (plugin_subparser_map .keys ()))
576549
577- if not parsed_plugin_args and not parsed_args .plugin_configs :
578- logger .info ("No plugins config args specified, running default config: %s" , DEFAULT_CONFIG )
579- plugin_configs = [DEFAULT_CONFIG ]
580- else :
581- plugin_configs = parsed_args .plugin_configs or []
550+ parsed_args = parser .parse_args (top_level_args )
582551
583- system_info = get_system_info (parsed_args )
584- log_system_info (log_path , system_info , logger )
552+ if parsed_args .log_path and parsed_args .subcmd not in ["gen-plugin-config" , "describe" ]:
553+ log_path = os .path .join (
554+ parsed_args .log_path ,
555+ f"scraper_logs_{ datetime .datetime .now ().strftime ('%Y_%m_%d-%I_%M_%S_%p' )} " ,
556+ )
557+ os .makedirs (log_path )
558+ else :
559+ log_path = None
560+
561+ logger = setup_logger (parsed_args .log_level , log_path )
562+ if log_path :
563+ logger .info ("Log path: %s" , log_path )
564+
565+ if parsed_args .subcmd == "describe" :
566+ parse_describe (parsed_args , plugin_reg , config_reg , logger )
567+
568+ if parsed_args .subcmd == "gen-plugin-config" :
569+ parse_gen_plugin_config (parsed_args , plugin_reg , config_reg , logger )
570+
571+ parsed_plugin_args = {}
572+ for plugin , plugin_args in plugin_arg_map .items ():
573+ try :
574+ parsed_plugin_args [plugin ] = plugin_subparser_map [plugin ][0 ].parse_args (plugin_args )
575+ except Exception as e :
576+ logger .error ("%s exception parsing args for plugin: %s" , str (e ), plugin )
577+
578+ if not parsed_plugin_args and not parsed_args .plugin_configs :
579+ logger .info (
580+ "No plugins config args specified, running default config: %s" , DEFAULT_CONFIG
581+ )
582+ plugin_configs = [DEFAULT_CONFIG ]
583+ else :
584+ plugin_configs = parsed_args .plugin_configs or []
585585
586- plugin_executor = PluginExecutor (
587- logger = logger ,
588- plugin_configs = get_plugin_configs (
586+ plugin_config_inst_list = get_plugin_configs (
589587 plugin_config_input = plugin_configs ,
590588 system_interaction_level = parsed_args .sys_interaction_level ,
591589 built_in_configs = config_reg .configs ,
592590 parsed_plugin_args = parsed_plugin_args ,
593591 plugin_subparser_map = plugin_subparser_map ,
594- ),
592+ )
593+
594+ system_info = get_system_info (parsed_args )
595+ log_system_info (log_path , system_info , logger )
596+ except Exception as e :
597+ parser .error (str (e ))
598+
599+ plugin_executor = PluginExecutor (
600+ logger = logger ,
601+ plugin_configs = plugin_config_inst_list ,
595602 connections = parsed_args .connection_config ,
596603 system_info = system_info ,
597604 log_path = log_path ,
0 commit comments