Validate option / minor

Author: daniil-lyakhov

examples/openvino/aot/aot_openvino_compiler.py

@@ -51,11 +51,11 @@ def load_model(suite: str, model_name: str):
         raise ValueError(msg)
 
 
-def load_calibration_dataset(dataset_path: str, suite: str, model: torch.nn.Module):
+def load_calibration_dataset(dataset_path: str, suite: str, model: torch.nn.Module, model_name: str):
     val_dir = f"{dataset_path}/val"
 
     if suite == "torchvision":
-        transform = torchvision_models.get_model_weights(model.name).transforms()
+        transform = torchvision_models.get_model_weights(model_name).DEFAULT.transforms()
     else:
         transform = create_transform(**resolve_data_config(model.pretrained_cfg, model=model))
 
@@ -87,14 +87,16 @@ def dump_inputs(calibration_dataset, dest_path):
     return input_files, targets
 
 
-def main(suite: str, model_name: str, input_shape, quantize: bool, dataset_path: str, device: str):
+def main(suite: str, model_name: str, input_shape, quantize: bool, validate: bool, dataset_path: str, device: str):
     # Ensure input_shape is a tuple
     if isinstance(input_shape, list):
         input_shape = tuple(input_shape)
     elif not isinstance(input_shape, tuple):
         msg = "Input shape must be a list or tuple."
         raise ValueError(msg)
 
+    calibration_dataset = None
+
     # Load the selected model
     model = load_model(suite, model_name)
     model = model.eval()
@@ -114,7 +116,7 @@ def main(suite: str, model_name: str, input_shape, quantize: bool, dataset_path:
         if not dataset_path:
             msg = "Quantization requires a calibration dataset."
             raise ValueError(msg)
-        calibration_dataset = load_calibration_dataset(dataset_path, suite, model)
+        calibration_dataset = load_calibration_dataset(dataset_path, suite, model, model_name)
 
         captured_model = aten_dialect.module()
         quantizer = OpenVINOQuantizer()
@@ -146,12 +148,15 @@ def transform(x):
     exec_prog = lowered_module.to_executorch(config=executorch.exir.ExecutorchBackendConfig())
 
     # Serialize and save it to a file
-    model_name = f"{model_name}_{'int8' if quantize else 'fp32'}.pte"
-    with open(model_name, "wb") as file:
+    model_file_name = f"{model_name}_{'int8' if quantize else 'fp32'}.pte"
+    with open(model_file_name, "wb") as file:
         exec_prog.write_to_file(file)
-    print(f"Model exported and saved as {model_name} on {device}.")
+    print(f"Model exported and saved as {model_file_name} on {device}.")
+
+    if validate:
+        if calibration_dataset is None:
+            calibration_dataset = load_calibration_dataset(dataset_path, suite, model, model_name)
 
-    if quantize:
         print("Start validation of the quantized model:")
         # 1: Dump inputs
         dest_path = Path("tmp_inputs")
@@ -172,18 +177,17 @@ def transform(x):
         subprocess.run(
             [
                 "../../../cmake-openvino-out/examples/openvino/openvino_executor_runner",
-                f"--model_path={model_name}",
+                f"--model_path={model_file_name}",
                 f"--input_list_path={inp_list_file}",
                 f"--output_folder_path={out_path}",
             ]
         )
 
         # 3: load the outputs and compare with the targets
-
         predictions = []
         for i in range(len(input_files)):
             tensor = np.fromfile(out_path / f"output_{i}_0.raw", dtype=np.float32)
-            predictions.append(torch.tensor(np.argmax(tensor)))
+            predictions.append(torch.argmax(torch.tensor(tensor)))
 
         acc_top1 = accuracy_score(predictions, targets)
         print(f"acc@1: {acc_top1}")
@@ -207,6 +211,11 @@ def transform(x):
         help="Input shape for the model as a list or tuple (e.g., [1, 3, 224, 224] or (1, 3, 224, 224)).",
     )
     parser.add_argument("--quantize", action="store_true", help="Enable model quantization.")
+    parser.add_argument(
+        "--validate",
+        action="store_true",
+        help="Enable model validation. --dataset argument is requred for the validation.",
+    )
     parser.add_argument("--dataset", type=str, help="Path to the calibration dataset.")
     parser.add_argument(
         "--device",
@@ -219,4 +228,4 @@ def transform(x):
 
     # Run the main function with parsed arguments
     with nncf.torch.disable_patching():
-        main(args.suite, args.model, args.input_shape, args.quantize, args.dataset, args.device)
+        main(args.suite, args.model, args.input_shape, args.quantize, args.validate, args.dataset, args.device)

examples/openvino/executor_runner/openvino_executor_runner.cpp

@@ -82,7 +82,6 @@ void dump_outputs(Result<Method> &method, const char *output_folder_path,
     std::ofstream fout(output_file_name.c_str(), std::ios::binary);
     fout.write(output_tensor.const_data_ptr<char>(), output_tensor.nbytes());
     fout.close();
-    ET_LOG(Info, "Write outputs to file %s", output_file_name.c_str());
   }
 }
 
@@ -135,8 +134,6 @@ ProcessInputsResult process_inputs(Result<Method> &method,
             method_meta.input_tensor_meta(input_index);
         auto input_data_ptr = inputs[input_index].toTensor().data_ptr<char>();
 
-        ET_LOG(Info, "Read inputs from file %s",
-               input_files[input_index].c_str());
         std::ifstream fin(input_files[input_index], std::ios::binary);
         fin.seekg(0, fin.end);
         size_t file_size = fin.tellg();