ut

Author: xadupre

_unittests/ut_torch_models/test_test_helpers.py

@@ -108,6 +108,28 @@ def test_validate_model_onnx_dynamo_os_ort(self):
         onnx_filename = data["onnx_filename"]
         self.assertExists(onnx_filename)
 
+    @requires_torch("2.7")
+    @hide_stdout()
+    @ignore_warnings(FutureWarning)
+    @requires_experimental()
+    def test_validate_model_custom_os_ort(self):
+        mid = "arnir0/Tiny-LLM"
+        summary, data = validate_model(
+            mid,
+            do_run=True,
+            verbose=10,
+            exporter="custom",
+            dump_folder="test_validate_model_custom_os_ort",
+            patch=True,
+            stop_if_static=2 if pv.Version(torch.__version__) > pv.Version("2.6.1") else 0,
+            optimization="default+os_ort",
+        )
+        self.assertIsInstance(summary, dict)
+        self.assertIsInstance(data, dict)
+        self.assertLess(summary["disc_onnx_ort_run_abs"], 1e-4)
+        onnx_filename = data["onnx_filename"]
+        self.assertExists(onnx_filename)
+
     @requires_torch("2.7")
     @hide_stdout()
     @ignore_warnings(FutureWarning)

test_validate_model_custom_os_ort/arnir0_Tiny-LLM-custom-default+os_ort/arnir0_Tiny-LLM-custom-default+os_ort.stats

@@ -0,0 +1,88 @@
+:disc_onnx_ort_run_abs:8.344650268554688e-07;
+:disc_onnx_ort_run_dnan:0;
+:disc_onnx_ort_run_n:204672.0;
+:disc_onnx_ort_run_rel:0.00033429367266728856;
+:disc_onnx_ort_run_sum:0.018881732111708516;
+:disc_patched_abs:0;
+:disc_patched_dnan:0;
+:disc_patched_n:204672.0;
+:disc_patched_rel:0;
+:disc_patched_sum:0.0;
+:dump_folder:test_validate_model_custom_os_ort/arnir0_Tiny-LLM-custom-default+os_ort;
+:dump_folder_name:arnir0_Tiny-LLM-custom-default+os_ort;
+:export_args:();
+:export_exporter:custom;
+:export_kwargs:dict(input_ids:T7s2x3,attention_mask:T7s2x33,position_ids:T7s2x3,past_key_values:DynamicCache(key_cache=#1[T1s2x1x30x96], value_cache=#1[T1s2x1x30x96]));
+:export_optimization:default+os_ort;
+:export_strict:False;
+:model_class:LlamaForCausalLM;
+:model_config:{'vocab_size':32000,'max_position_embeddings':1024,'hidden_size':192,'intermediate_size':1024,'num_hidden_layers':1,'num_attention_heads':2,'num_key_value_heads':1,'hidden_act':'silu','initializer_range':0.02,'rms_norm_eps':1e-05,'pretraining_tp':1,'use_cache':True,'rope_theta':10000.0,'rope_scaling':None,'attention_bias':False,'attention_dropout':0.0,'mlp_bias':False,'head_dim':96,'return_dict':True,'output_hidden_states':False,'output_attentions':False,'torchscript':False,'torch_dtype':'float32','use_bfloat16':False,'tf_legacy_loss':False,'pruned_heads':{},'tie_word_embeddings':False,'chunk_size_feed_forward':0,'is_encoder_decoder':False,'is_decoder':False,'cross_attention_hidden_size':None,'add_cross_attention':False,'tie_encoder_decoder':False,'max_length':20,'min_length':0,'do_sample':False,'early_stopping':False,'num_beams':1,'num_beam_groups':1,'diversity_penalty':0.0,'temperature':1.0,'top_k':50,'top_p':1.0,'typical_p':1.0,'repetition_penalty':1.0,'length_penalty':1.0,'no_repeat_ngram_size':0,'encoder_no_repeat_ngram_size':0,'bad_words_ids':None,'num_return_sequences':1,'output_scores':False,'return_dict_in_generate':False,'forced_bos_token_id':None,'forced_eos_token_id':None,'remove_invalid_values':False,'exponential_decay_length_penalty':None,'suppress_tokens':None,'begin_suppress_tokens':None,'architectures':['LlamaForCausalLM'],'finetuning_task':None,'id2label':{0:'LABEL_0',1:'LABEL_1'},'label2id':{'LABEL_0':0,'LABEL_1':1},'tokenizer_class':None,'prefix':None,'bos_token_id':1,'pad_token_id':None,'eos_token_id':2,'sep_token_id':None,'decoder_start_token_id':None,'task_specific_params':None,'problem_type':None,'_attn_implementation_autoset':True,'transformers_version':'4.52.0.dev0','model_type':'llama'};
+:model_config_class:LlamaConfig;
+:model_expected:CausalLMOutputWithPast(logits:T1s2x3x32000,past_key_values:DynamicCache(key_cache=#1[T1s2x1x33x96], value_cache=#1[T1s2x1x33x96]));
+:model_id:arnir0/Tiny-LLM;
+:model_inputs:dict(input_ids:T7s2x3,attention_mask:T7s2x33,position_ids:T7s2x3,past_key_values:DynamicCache(key_cache=#1[T1s2x1x30x96], value_cache=#1[T1s2x1x30x96]));
+:model_inputs_opionts:;
+:model_nweights:12988992;
+:model_shapes:str;
+:model_size:51955968;
+:model_task:text-generation;
+:onnx_filename:test_validate_model_custom_os_ort/arnir0_Tiny-LLM-custom-default+os_ort/arnir0_Tiny-LLM-custom-default+os_ort.onnx;
+:onnx_opt_optimized:1;
+:onnx_ort_inputs:dict(input_ids:A7s2x3,attention_mask:A7s2x33,position_ids:A7s2x3,past_key_values_key_cache_0:A1s2x1x30x96,past_key_values_value_cache_0:A1s2x1x30x96);
+:onnx_size:43892;
+:op_opt_all_added:117;
+:op_opt_all_removed:218;
+:op_opt_all_time_in:0.41514878199086525;
+:op_opt_cst_added:0;
+:op_opt_cst_removed:0;
+:op_opt_cst_time_in:0.0;
+:op_opt_export_optimization:0.41514878199086525;
+:op_opt_fused_attention:0;
+:op_opt_fused_bias_gelu:0;
+:op_opt_fused_cos_sin_cache:0;
+:op_opt_fused_erf_gelu:0;
+:op_opt_fused_gelu:0;
+:op_opt_fused_gqa:0;
+:op_opt_fused_mha:0;
+:op_opt_fused_packed_qkv_for_gqa:0;
+:op_opt_fused_partial_rotary_embedding:0;
+:op_opt_fused_rms_normalization:0;
+:op_opt_fused_rotary_embedding:0;
+:op_opt_fused_sdpa:1;
+:op_opt_fused_skip_layer_normalization:0;
+:op_opt_fused_skip_rms_normalization:0;
+:op_opt_max_iter:13;
+:op_opt_n_applied:49;
+:op_opt_unique_applied:24;
+:op_opt_unique_matched:48;
+:time_create:5.664649608999753;
+:time_export_export_onnx_c:7.78976812899964;
+:time_export_optimization:0.41514878199086525;
+:time_onnx_save:0.21937757000159763;
+:time_optim_os_ort:0.2757421719998092;
+:time_optim_to_ir:0.004986361000192119;
+:time_run:0.04424335000112478;
+:time_run_patched:0.006461869001213927;
+:time_time_onnx_ort_create:0.10978344900104275;
+:time_time_onnx_ort_run:0.14714581799853477;
+:version_date:2025-04-29T11:56:42;
+:version_device:;
+:version_do_run:True;
+:version_drop_inputs:[];
+:version_dtype:;
+:version_dump_folder:test_validate_model_custom_os_ort;
+:version_exporter:custom;
+:version_model_id:arnir0/Tiny-LLM;
+:version_numpy:2.2.5;
+:version_onnx:1.19.0;
+:version_onnx_diagnostic:0.4.4;
+:version_onnxruntime:1.22.0+cu126;
+:version_onnxscript:0.3.0.dev20250301;
+:version_optimization:default+os_ort;
+:version_ortfusiontype:;
+:version_patch:True;
+:version_quiet:False;
+:version_stop_if_static:2;
+:version_torch:2.8.0.dev20250423+cu126;
+:version_trained:False;
+:version_transformers:4.52.0.dev0;