Use save/load in the MIGraphX EP

Author: Ted Themistokleous

onnxruntime/core/providers/migraphx/migraphx_execution_provider.cc

@@ -1151,32 +1151,48 @@ Status MIGraphXExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>&
     migraphx::program prog;
 
     if (!no_input_shape) {
-      prog = migraphx::parse_onnx_buffer(onnx_string_buffer, options);
-      if (fp16_enable_) {
-        migraphx::quantize_fp16(prog);
-      }
 
-      // Read in the calibration data and map it to an migraphx paramater map for the calibration ops
-      if (int8_enable_ && int8_calibration_cache_available_) {
-        migraphx::quantize_int8_options quant_opts;
-        migraphx::program_parameters quant_params;
 
-        auto param_shapes = prog.get_parameter_shapes();
+      if(!load_compiled_model)
+      {
+        prog = migraphx::parse_onnx_buffer(onnx_string_buffer, options);
+        if (fp16_enable_) {
+          migraphx::quantize_fp16(prog);
+        }
 
-        for (auto&& name : param_shapes.names()) {
-          auto dynamic_range_i = dynamic_range_map.find(name);
-          if (dynamic_range_i != dynamic_range_map.end()) {
-            quant_params.add(name, migraphx::argument(param_shapes[name], &(dynamic_range_i->second)));
+        // Read in the calibration data and map it to an migraphx paramater map for the calibration ops
+        if (int8_enable_ && int8_calibration_cache_available_) {
+          migraphx::quantize_int8_options quant_opts;
+          migraphx::program_parameters quant_params;
+
+          auto param_shapes = prog.get_parameter_shapes();
+
+          for (auto&& name : param_shapes.names()) {
+            auto dynamic_range_i = dynamic_range_map.find(name);
+            if (dynamic_range_i != dynamic_range_map.end()) {
+              quant_params.add(name, migraphx::argument(param_shapes[name], &(dynamic_range_i->second)));
+            }
           }
-        }
 
-        quant_opts.add_calibration_data(quant_params);
-        // perform static quantization on the programs
-        migraphx::quantize_int8(prog, t_, quant_opts);
+          quant_opts.add_calibration_data(quant_params);
+          // perform static quantization on the programs
+          migraphx::quantize_int8(prog, t_, quant_opts);
+        }
+        migraphx::compile_options co;
+        co.set_fast_math(false);
+        prog.compile(t_, co);
+        if (save_compiled_mode)
+        {
+          migraphx::file_options fo;
+          fo.set_file_format("msgpack");
+          migraphx::save(prog, save_compiled_path, fo);
+        }
       }
-      migraphx::compile_options co;
-      co.set_fast_math(false);
-      prog.compile(t_, co);
+      else
+      {
+        prog = migraphx::load(load_compiled_path);
+      }
+
       auto prog_output_shapes = prog.get_output_shapes();
       for (std::size_t i = 0; i < output_names.size(); ++i) {
         auto out_len = prog_output_shapes[i].lengths();
@@ -1196,7 +1212,9 @@ Status MIGraphXExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>&
       *p = {context->allocate_func, context->release_func, context->allocator_handle, map_progs_[context->node_name],
             map_onnx_string_[context->node_name], options, t_, map_input_index_[context->node_name], &mgx_mu_,
             map_no_input_shape_[context->node_name], fp16_enable_, int8_enable_,
-            int8_calibration_cache_available_, dynamic_range_map, dump_model_ops_};
+            int8_calibration_cache_available_, dynamic_range_map,
+            save_compiled_mode_, save_compiled_path_,
+            load_compiled_mode_, load_compiled_path_, dump_model_ops_};
       *state = p.release();
       return 0;
     };
@@ -1270,33 +1288,47 @@ Status MIGraphXExecutionProvider::Compile(const std::vector<FusedNodeAndGraph>&
       // input shapes are different, needs to re-parse onnx and
       // re-compile the program
       if (!input_shape_match) {
-        prog = migraphx::parse_onnx_buffer(onnx_string, cmp_options);
-        if (fp16_enable) {
-          migraphx::quantize_fp16(prog);
-        }
+        if(!load_compiled_model)
+        {
+          prog = migraphx::parse_onnx_buffer(onnx_string, cmp_options);
+          if (fp16_enable) {
+            migraphx::quantize_fp16(prog);
+          }
 
-        // Read in the calibration data and map it to an migraphx paramater map for the calibration ops
-        if (int8_enable && int8_calibration_cache_available) {
-          migraphx::quantize_int8_options quant_opts;
-          migraphx::program_parameters quant_params;
+          // Read in the calibration data and map it to an migraphx paramater map for the calibration ops
+          if (int8_enable && int8_calibration_cache_available) {
+            migraphx::quantize_int8_options quant_opts;
+            migraphx::program_parameters quant_params;
 
-          auto param_shapes = prog.get_parameter_shapes();
+            auto param_shapes = prog.get_parameter_shapes();
 
-          for (auto&& name : param_shapes.names()) {
-            auto dynamic_range_i = map_dynamic_range.find(name);
-            if (dynamic_range_i != map_dynamic_range.end()) {
-              quant_params.add(name, migraphx::argument(param_shapes[name], &(dynamic_range_i->second)));
+            for (auto&& name : param_shapes.names()) {
+              auto dynamic_range_i = map_dynamic_range.find(name);
+              if (dynamic_range_i != map_dynamic_range.end()) {
+                quant_params.add(name, migraphx::argument(param_shapes[name], &(dynamic_range_i->second)));
+              }
             }
+
+            quant_opts.add_calibration_data(quant_params);
+            // perform static quantization on the programs
+            migraphx::quantize_int8(prog, t, quant_opts);
           }
 
-          quant_opts.add_calibration_data(quant_params);
-          // perform static quantization on the programs
-          migraphx::quantize_int8(prog, t, quant_opts);
+          migraphx::compile_options co;
+          co.set_fast_math(false);
+          prog.compile(t, co);
+          if (save_compiled_mode)
+          {
+            migraphx::file_options fo;
+            fo.set_file_format("msgpack");
+            migraphx::save(prog, save_compiled_path, fo);
+          }
+        }
+        else
+        {
+          prog = migraphx::load(load_compiled_path);
         }
 
-        migraphx::compile_options co;
-        co.set_fast_math(false);
-        prog.compile(t, co);
         mgx_state->prog = prog;
         param_shapes = prog.get_parameter_shapes();
         no_input_shape = false;