From f949039360bcdb3ca16e7e302dd601558b4faa4e Mon Sep 17 00:00:00 2001 From: YiyanZhai Date: Fri, 24 Oct 2025 16:49:05 -0400 Subject: [PATCH 1/4] clean --- examples/res/win_at_p_curve_gemm_o3_gpt-5.png | Bin 47758 -> 0 bytes examples/win_at_p.py | 552 ------------------ 2 files changed, 552 deletions(-) delete mode 100644 examples/res/win_at_p_curve_gemm_o3_gpt-5.png delete mode 100644 examples/win_at_p.py diff --git a/examples/res/win_at_p_curve_gemm_o3_gpt-5.png b/examples/res/win_at_p_curve_gemm_o3_gpt-5.png deleted file mode 100644 index 327511eb30d6d29413f9e5dcd5938cafd0b0afbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47758 zcmb@u2T+ySwl%uZR=Y9Kt)ha6jc$}AC<2N^wNW;RWRPqi2nZ-i$(X5aAN?tAWi|6jLW)uZYhTG)JHg*nFLFqBowD-}IurT6|EBLM z^8F3%nlbrKIlhg$lzczGmWz#i_k8fbUZSw2VOeW0yQoD(TAGNcO><*IB>Btw9G47> z>bS_cnF*Kd^>1uTsM<{C=H^9(g+q?y(wC(VLO2Y)rC$5T324c8><<>>#0 z-~O3ySzFe?P2tQx_W%DA|L-q7E4AmWscBU1Ous~;QJ!H#hE1)J$6O>{aM>@v@KC9L zef#$9)}1>Vfr18Aahm+fe&mfQvavcF=ytZYX(!(<$yu16x#i=dZE9+Iv){5h&Oa$< zT=y=AxLRkJN3LcE|3z-2dE3XkxD6(!ruFSr(bMmL9qX-2 ztva*ifW5rkkCHEV;N-iT_bJdVs$!n7NKeXMUBM!#=p-F!knKp1NYu-4&cqI(c!nu| zYbbi%=$!AhI!-SmDe8DgT;BY&;y~_<^`j*z)aqm-BNoX)PxslOqjQ7#ELpR|#s2>O zAB+x=cbC1+H)7j^m$n%F`GuL8IVFYDp}UG^dv0IjH)e`=k3NbP~Gz1MB0Pv6PlC-dORS2*m;v3WB;O*h>_^X9Z=e~2#ma5!_SB(>#YiyQN=zkaqWmx^x6z8E$(9T7p5 znrO6(Q+?*upSR#1UmVNXds*%M?-OU7>Y8I2K{;-B@nKsH4L@wFEs;@Caf*S0(^%NI z^_@ck1EVwTI^!&BlN1ycD=}*8+1Sh-#H$NfD_v?2(>6ylTqYh!C1Jv=N>iWxvW0Hoj1tRHR9nC9bjRq zrhhF%k$mJNlTQgXU-6tHEn2|AXhETu4M7sU1?)ZBTCrxfph=J$V(_Wx^&`+r>`&ewN#sxz$EG_|m$#VFORJn4qdFN~X4@lmM@ z6L|~%=0Qc}<+Urex~lxKm1Hzz2Pa8tbY!IYw|CctR~@GN`TB;FQ-uqq3Znr4YgP{;5sfb*a%wxziSpq6+Dj zlN{NlbIBD?-@6rkDjQK_M7g4(XIxW=)VjH@nbaa7X(W-(G!-req_T>R9!o^YioU!p1j_RaOOy+euD+A(x~01r;0`9>`lra_Fg{ z@tv^fYuC4(E7N0~5@bJhlU9rY~%RqBZc957=x8rKYul+9< z97?f^9n;h{h{R5Kb#JrRlP8D0i$s*%5FLujJHH(vpYtqwyPeBvZhEXf*DX6DFHn2q z4@t+LpSgA!WL2%>W?V_dg7de+GH+Xw7HwK`qeBL&Vjq-A3`o)hG8rIAgEPRpeRkHh zG((tp#zFe3NIm3%{W3|O1O1n-xm&FWe1u`ZL8K0Bh*4F0G`i6-nCzS4J8>)y+ zd}Z0n1VKi4@>D~_kOXIXO$!9r*}%VV}}>#bY2Y_nG|9>*p{glcAL+FhwE;~@2R z`dqMLQs6GVD3Zi<2yUfNB2CTZ4b#rvS3{5saf(f^bf`7hNdu1hf+;#{!s}Uk*Xa2N zU?qIk$pEQ>ks_;_X@cmsET zW!2lhj^`Wa1{s;v-zSnQQ!`!*xJ@&*KeL^#m>|G3$nM<~bx>6)jpssG&k0AvT#iMI zsCoV9G<9ipb;PS*GbUmmiT~HCQz))@WK8MLQd06gC7=KFv8AnkL`E?gu=5*As7p7b zG|U=r>MuB^9+c|1Z*LKPF3Tz_zgz(E1WD5A{y}l>y#*3- zz1vmg>t?zdj-GD1cU1Xi?W<*vb2M1-DNp*J)R}FMo95;Mrqmp>E?UG!VO77gsNQdN zDq(W?sBLhro1KJkW5XY0xsOC-^l*LP`&1)a)&CqedI#53s7wF- zChOYKu1bf=!Bg?xE8-#wFWjGYIxXWiBcISP^pkgpJ>z6*U>ret=lKwnfip*cJhtf` z{FdL1{f9-AF#l_~g6&r;=6KDvG&Yk((BSXg!P3vjM_scc<+T9q)zQjZMXl;JR8>_8 zo;xR|_hN$G>j~Rl(-Q`3Rm}GX8hQ-QS=a8V^h+5T?ob)>*hgMdV^zM0Qe_d--0ZAz z@BvLNW+$hTn2@-CuwuY$)Sb49stq1_9>#&~zrIKl^lQ58cT7(C=EWQ%%#dh;dgX0a ztP(%{NAA+g;(jd-)KjFJ2bs7tRUa&+Oz4~$-$6E^|J4pIaT%FBeWCCJJaN0$yF4H( zYhdKlF=5ZWcKu3%C7JsejAY_>I+M^Nj*YSDJ`>{%l0sqTqAoSG?`~$WqvnngE^$#& z&2>ADsTdnaT)A>(JqJfZS%`!sa8prPnPBbLkDvc~Hz$G`+K?=4%lj8{TJFK=%AF^S+gGyU1l^HZIo&ZB4f^SCEpl>|#JlUVZF zfP^v?tEt{&w$|23%hzzk0HlaoH=ZaC*rl$k8&pwQDSEMAzdFf458$W@6Rs#V?#HXH z&ol*MP!lTU;ujPYcjwM3LVudF9MqAoKi*+`1_=12+h(=&9W52y!1LnNIr&gg%K$MO z!CyBMcnAQzorb(s2|hB0rKCX5ogG&EP*4zP0is~0NI!l0^k{c{a-4Z(RMHCxhwg}S znFalW2M>Z(VC*uw^{%iyWhDq%C3DhVJ|N=j*Aw6~9cd%A1{XtgU!LRb8u+ZI<}o)G zNbP!hTaa(q_5AtgWikt{vynjX>^Swv^VV1nLEX$qzvM< zlLM$@11))>%L2+P66 zC0%ROFHc*<{&A2>+@4j3**7%lKWN*QD9D1V$7s3Q?r>^&X@z_rNR|8Ruk|h=BaJ}d z4xlG%eLV2sG`d!jeymG>=24f4et$ru*wvfGeiEvMXReA=3>5BjIx?mnZTiN%I7u;I zL??=5y%TSD9v;qho6$Ftm{_?naxnAVL~tp9<}(6<-QC@t`_gNx5_I;GNffoL(TF}C zGD;ZZm!n%XU0q+Fdr6;oYlSs-KY5FUqu-@f8cJm*wH>RA>YS!(SP3fCI4qsnctBLN zdd%VE$&)W?S*5inLsN$^dwa_zpJ6b%jTWW@j|cDn_HttX-o5IDg@u5q5Awh9n2g0F zfy-F*dcKTg&O-2yGz=n-9DYffjV{%PgWbWZP@E@y__|-^W2ep9I^iV+I%~9 z98nAqxPQU+?m4U%f|(rp>NSBsDzR#x9Np@s0es@lmb*Y$F)i7oEi0IC@al5*J*Tf3Ui!Ao4MPzm<>Gk1mZ_ry>5eDv<_7NhoM>qYp*;Om z0+(bROsB49-`gS)_3+^)g5seBOq*O6v#S48p4_U}kYb|vqE@_I)S^lraBA0=uQCg> zsXaYfiMp&V20b;2+^i*9TO}_&|LcrP)y6Ym5R%4f`a{a+3EWUQ0BF-0>T;Y$qtPQc zrJR4JmP=Q@y}H65BxH7WHZ?cl_f4B(k_@sH=o^F#_1i_kwJ=Nc2rR>ZOXtic10oQe zSzVeyH}s8GEL-M($+h@vZ~s6>NnpxtBj+`^31v`L-tg7=C)i=<&Lw;YIQqb<9b=f| z%$nPHsr}P^dc?JBza$!DkEe}{{3s>k&qc%V{B^US@9T?aJv~rbaz0sBR8)w#jGv>a z&5ZW}ktDA5v0-xwU%%)5+zCU+549wMQ#5aMaMg+5c3qg82#h=WyqZk*r%&?OUU(&b zX=y{cUszbO)5wpmmkrhr$i_FyahY(;zWDCiZ$RK)r(Wf5joz?J_iXE%%k+qH>6v5u znlC+b?ET7BDdPX&!82^C7_;)wB#N}@osEIwMte$K!WeK=(-#RarJ=8=q(lSnkjR;( zF*`Sxe)h|@4W`8~>fJ9|+$IdhdTNzr<|m&? zPqr>~8+^NhdDSWnN5_}8c6J7}Qlp;_M+QYQfas;bj=hCMi=cKPnrg8aSU@8}rg;#0 z>HBYpGoN;^f*KX09(|l-fH5NGh=`+>hJwzlm91Igg5 z+WForwfkCJpDoOG%5Y95k%6Ryo_?q$r8XTwe&#&-I<8u6!D)*h#)VWm|x zJ-nbFRR4Wo!l#$c!4*(_y@*SE`%%?(fmzKvrR%ic-lGU_qFgK?g2UymB=_KDkU{F-L#ZFQ782Xk`+Dw zn+gDd!IMk`4dGS!-L`l*Lb5RC2v>r6v$iB)&?BB5_2J{pbwH%iy9wU?9}bDQbQ zNMdV;rpzak7w#Rum!lTZ4f0&vjlZ9re)JF#lR!c`nLi-BhTxUC=eFFsdgV&hv!g6y z{Y`oZ`dUaMQmV|%GpFx}7<*;JlPB?1C?58eB)a-Y={^M|CI8WjJTDvix+@}Y^RGv6 z`T*Xq(XFxb;eJ`tm1Cm__fJnVQ#uNwOqgW1!JUQ1O$w%r7UX%Xcabj zb3dHVYacpvg}{d}k9i%S(#nOomW4vo)L2Y;l2j4Ww||qhOz}_h8k9@-3Up;dJm#kk zEXI-WS2!>u{bhITPl5Pfy6iz?bsuOh7@#obQk&059YZ-rU)_z~A zFdDfO2(#)ii&O*_bkElpXLK`dBRSnhRf!}72_5weEJ;??H7dLA{Hs-H-YYPr-%d5J z+);aH7*WzMAYcd4DxXsP?l%3Hmq4O=$FAVBwVChU zzJhAr-ISdXUCiVfkZ)ZX9@wCh zACQ8zlMQo7P$LTjkSM7-l2Tnpv!OdX*dTbdzDEdh%98hM*jxI(c z!2oY#v)xJ1do|kM$Wg zVbU|bd)*LquNf=9SfRkzQPH(=p3A^~<)Yf4u{l5}L)v*Xh8jzIu*15bva*u!d|Jhg zKejiPHJ#62O^a(~0L16IE}royRO_YM=(J8#erKOgeIx zolec?>C2BinaOYDJ3n_jBi{4+N1?CZ`^cvD(H9Er zs7UXwO}23m=ebRB*3veCkZ9C{o$7Z#q?4f@d&Yght=FyFUMWHZ01Mz_dc@R7XSOMQ zW@5mjUP>>++Mf#g@~PX_)1u&1hPjI0%w8s7u$;vOn|^|mKJa?v_|(iy52h_^@ZCy0 zcofuL1$vOEWfa&{h@(%cmBwFxU90Y(R98gopJ!xQlT%Le^93^VPqYo4 zq?)ozyS}o_vpod@Qo$Xlzm0}>58L@S4EjrQEuA|S5iw~0h3`7a+0)(e4rA5YE^&(} zQK4kGqlR&P4bA^qT~~WFL2WoE#^2*hMM7w7)GsbzfSGV=k>rDOK~YByW3J_HYU^WG zISyblSR9<>7`hYq=(KK+yc14cH+?1y7cY4q`d7E*lW6Bj3Y@ZCxw4J5r{ zUFHPS)a<+#VI%g@TgEJ-$EA2gft4qW{>93PMoc7VP11pVCz@T7zuJ*f?!x?p3?Z2y zy{`#u0o8bA@Zj;|covyi6&9&cUqW+9c>{dSO{e&>KPh?nUOC_E7r5)DufFF08DQ^U zSG=_`-K2O!lXFiZl)jgw#7>rf^fF`G@We#z^Hce0Sg*^LFYgY}v;Km+pTk;d1A6}+tKwovX_hegEU_Yg(S54Tvc zj1k7UbEAQm>7;HO&k&_4;qS4%iKG-!k$%J0u;=wgatn&hb4e;FE{$FT#P;0<2xBLW zlx6;F)p)lT{{UFSF{;m~G?YD6C;FSl znl84my7XZ=SLL`|oP(%Ak3coF`k&9g5T2k3mJtmNc;Djfm1WcTNOhv#K0>E%-Mgm+ z9iN-FVhzXV`gF^&p5(l&{@b^2SEkn*TK>m7>(HfV2M;$nHdsM&cuDQTOgXH4e>}=u zfqvx9rY9NA&c*8@8Lx^RAe0dh;ps^sXal+W*6rKsi2gv34OsiSZNSxUF}+(8vi5hN5WVD@IfmLs zI_jQVq+L&Rb#*1})ZN&5b7$zFJxw-!53k@MEOMu8vLC~X22aS(qsl!x+2&)3jS-pU zvuFSq+~9Mqy{j_1=k1EEei5l<92-qHdHuV`1zH}=Vtl(X4*^ujOUGAk5crHDGgs`N zzYKhRaad+{@T#HPm?rqRJVV?p7Rvhxl=3`lk#Qjk_|8J`~>CGA`lVIiGaQ$d$ zCHC0RW6pv2ACN9HF&btL;~s0E57l?HTfx5aKhM^Z9M>&n^D2y*lY7!3-$ZHX@Y+qxY* z!oq5hZJF1s(Spwe+P{d2nOUIAECQ$H?Bq~4YEWPV{NRERSYnLw+?|m}AgY{sdy&bn z>A&_dTYyXxDf$8ZTYuXvB2qOr2KlM6Sy2uC8;0-^P?KU=D+r&7IzzR3^G4Rh#Kas# zS2xc+2dc`m$;rveZr)fDmxx3g2T zoBvx4oR1q?MSgxhH`$#RwygZ^H&tAnn+6TI&wOGQWqLaEp#)9nGom01+}zx;7ur5T zihcx*$`9@xQoRSKCv44&WvgM-P-p4mXaZUAPK+sa=+QDUHs(QG#!sMl`XBM(OvGL* z&fOU<@HLPuPru*}E<(!a5m1I-ZGonkff_1|IE|)vz`>G;Xtx;D(-Q6>F_|;cD%RNv zNKN;t*nQ^U6a>?7(~R7wA|bmeijEV{63J@#HEO9r&8{O#=~fLj4!L+-Qs5zqBx)4J zPX~VGdHi&oJK8vfp>Ofrg0RNs1!d7LqYZY445~STR#xkkl_JKUekVE_!TZnvsWhzT zR1=4iAW=p7`pui;1r4%-9p|%plGIUX;Et_gS>*z(g;2PMm_9%&1wCGPdWU2Qi90L2 z{U>`kRTR+q4)uIXRK)ZO(YqV+C^&LsAxHNMFxtXSW;~9^`+J0 z;gOZ~jS%@4jEy=ovDyhn;bwQiCLky<;!0%4<9#+YtM@gAf4sXn9*NnaCgG9Snl0VT zU9u*vj0$IF!=B2BEu_eSs#}<~n;2k~&`5rYLJ`|(?pWw19}Kl88zH%BUU%P-6c5{Y`cmch{RejxXOGs0RImEiHbn&(ifK zsJ-_0RqK9HpKrbz_UJ+M$rUUN=S;9b7%gcP^TcWC%5|wL!}#Mo>{Ri0Kc=0reJv^? zZU&$Xy>91mvTOm_G|NM!OoF0_^hh)glgTxd@))+0b}r?N+SnU|>*Z8!hcc5!I&y~e z{Nx#u{u~mBUj<~f3i5tmOP&#~S0!4jum1D}-b}ZB=)38|%N4f$N@AsXKBYR@dS_Oo z`R7~n&mSStD`@bNyfoli?)Tfd}7C9HL|nW*qh<~@GLk4Auff}eezUk zsFYq^U7ehvI+Qo!*TbgNgj^#q)4M}yzW=xEivxyOrMaJa#z7q0rOebWZM8$)^j1#N z`IdIXNzFOm*$P7EJLz?+^nY=4DR@got3_B%jfg0A9pUtpp&=PB?Gixk4)T0w`{FvvTTK7=uaE$Kp zrpJsF=gFvZ8bM#sP*bBHtJHvwCUWZCFK|C-kVQ#y<+(axrQrRlP?7 zh9>>Jh|=-hT_E`L^*pryi4c7N%owDQ;X}9SZjF3fbQV7wp zAXxQ~U-e}`Vp3}-peDsBhe;E(Nj$V3fRqX}nyUAjEh1)R z!D>Ez@OD7%jl=d*P*6|{lXla{cC@;F?OF`^iM+gutgL7ArMA^jXNyw?V&Ah~Di$lb4g#4&VHwd8{lDcyUU|J zy>;N6V31ehsr~K}Tzg<(naa>&lD+gNb)v69JIcHpE~r`zZkh8O{-4w$i+8Ug$|r>S z@J+pnM3Xsg<|LmBI@ED~zaMHrfOY!Hl0(<9>s{&{Na%faq0Jt-jUUx`6f+j4Yx5EW zSIF+URUa5kf7JGOUV95J{p!M*ocZY<>^B4!XGjFqubu2rEIRb%%NK&>u}iD)6RDAs zy|4h#sX0R*H$$3#0rGo zxeQShNWC7qNp#O8dHEaC(?7Sb-YVS)l)rx|GN9Kt_j6_SO#<0?LD}?<`@RA+Qt|Qk z^NRogj`vuYx2R8xCEi`QJBWafteR)&+{FzQQ1P(WD)AXlia_W8$YF6p9?EMBA>tBc zG7Gk6si{4bqEr9oo$omY1YEW{a6YDa`7n%^o-3+WJ;{VG{2nN5AaRl=rrR`t7d?Toehy2i52|5$Lf?9K7Fve$X-XLWr7C-Y1Zp|Vq>kr@E@>y$5cH+<5_@pOal74YwNY1@PSEj)=W)d-b=lXAt69dGc zgGnf@th{rD$(veYPx{Ja)pX^&h3UP9ik`on7^*}tZoMG+YGvfn%{*a0o;R%T{q=l~ zMXf!{`!DL>MXleO3TcwM*rg3zY-UF{$5@nRYG*zdx4bx#Mv6g`REYDguC9*;a^UtA zNcbgyH*U4Y^eNu9$rhnQspYo)nt~-F$~J$f!$tf>#ckd_#JtU@=xGhG-e*+wR&Cr=A@?kT-zX;Lmde{T5ypJZ3We^!9U<3q z)r*{*T0i(Z5VE?OT2-EhM<1Z+`w7YMuV z3P+eqY(T1UboPpG3G2H)98$E{TE{JZi0`$4M(~~9*QfIbV*8%vx>3i|qR;5PwH|qp zZ=LF@kH{qK-`%PH=Cn!w1N~s=2PYP0S{Cd}MQhIjBxhk)!ZOoXndE|d!6yUvNrkqN zo1APCbMIyPWJ~M;yRTNZF~&^k=TE*|>k1^70D6+lTCcKXg?J$U zh@i;IsBfp+cd5a|(SU6lBP*AFd-#`eNzmpu>XJ1(r|wDge?H?QwF8;SZQ4F(v_g>{ zfv}{Es;Cb&7NsHL?F6xr)Lib#tWob)-0?Qlj4Y=s9mR<&T@Y^4&B{4RI|8Yly3dVi z=YVD+7?H{{Xo6<4p_Fm}CH#ug;`)_Zds4Y+Bsl2NnJKg2-&~&HHC)4vBI1-j+;fPh(Cbewl7CGG*Xk+$p_ zW@a^r^q?oaGBQ3FBjiMV;g&$nja(UO`)A-C*57|u`N3v_2&AIo3%XnN(n>Gh`-q7# zi{Cb{_1DfskLIfx@_N{vNnMxl&93ZaV4X;$UkL}gjNWt-D>~mNW=IJm@_`&0Mo@py ztf2-~DGduUWdDUj@Dus(r0z4X_zMj-+6=4Xvy`mUK`!#{F9eE@uI_rOTH!BrLTiHx zTX0t3RFzv137E5Mi3`zH&%z>($}1w`=ef!UmMq|g0~fwugGm;w?V0f1GgH%Ez?<^) zngpFQW1nLZ;3e;vD<~u729sz#)jw2Xp-{pqhiI&FB9Z;TnAuePW37gfYiE$;gg=V%~4rx zFm+{gz^Kv3ZO=!Ebc&&{``ZnRI<3>$9c(=5L2rpe2;<5@sI;#E5L~;O?s8gw#KmkNquJfe1*avhZ}ufw6jZ>5Ll_P zF5?X2I-x13ao=Uzdt_m5z$4vh#B__e?O#x*$cg|DNgxFQShUD!|L9Z8x#xSQsO^?{ z|3Io*#GST&A=AKbhyQW3t0)EAm5H{I1O>}*V$!#J}pol|mPG66b1tK29e*?Y~tthLh`N|{Zw(4NGknUfkP{qAr z%a+8SKYv!|kkJex1_wh7Om1Q69hE5dqKLcGs_xwbgLo-z37$5A zjTPGnpsTLUTbQ$f{A)2kOS)$A*26m1R^Vd^rJb0AK^^0(s$jHsf&qUHAD~A-Np^bL zOA1WEj~>NB2JJ$v7^w(Am=tJ;p%u+Be6ZM(jyvKyiB#N(qygQ?;&b>xpyQM*4h>Q5 z+&`sqm7R?PXL>G|_Sb&d#w>k93~;RV-2A}8ymDoY!Fyp>Vy+<5xq~$Ekb@(PBqobqtsBrvjb{QBO%}|Zi$!(klFRg)qXtK!*c9sZm z(AhLgsCSWgi>{BGZK{Y#^2M0~&{+d#BvboxsR_89Ar}T2bq1(QGKSat(I=A9+3HLJb&a;SfM4%Ih&4|Os7DE3RY&jS-x#jKZ zQb22@ioZcnKL#x(e!h2}itGJl^=M%NUigp=QJ4AN*DbdoNclzYYgcjNao3RHeUy1C z=v7sN$qB)s!dzy@Urq?ufUA)PLR!VO-`3rF{exJA$taM4BwO$HszPYonxKFFAt8xp z3hvS9C{R9uT&{}3OF)&bKgP8_Gh-5kLG)^sS=I&K*?S;eZhh<0vrDmV6^MU{&TQ(6 zvd`?cPzphOk}(0fThRmAfw0yaw5F>hTTe z9D$Re*v3cc_Op{00565k7zw-nK7*!L*$ZnpC3$GzIbCeY&@~=4jG3u{I~&-QicZhZ z3^-sce}e1@I0j^8J$KD?0KvM)A^sFM$2AcTa*1Ri-p*AH3JSOI1}C~_+^z%Q ztys0`Pas1$CDE=jkx*NXrWM2xQX}|&iywqE*UPPN@F0WIa$kTGjK=rtTjK4wU)$c% zK~xrc74&Phy9*f7SGm9hd}sVM!xqz=D&PS?D@t-A==6l|NLO53JjoJL87a|$Sn6Y@ zQD22ky&AF53WMUnxRb2tg*uZ!UJ!h>A>|BrQfap_ddr-SAkpj~xo@JRFP#)C35=!N z4}I9kzx(-gQcap9)f>a+u`m;M?&X!pE2u)M`fy6EU%x)WYxU;1Q~6#5Kj0^$2t3MB z=0#||`kMk*cf6@xpDrXW(S-DaFV;`r&cAv$L~58#&^3}eCE#W0L4Rw_QWnWSY3v8= zWAtXC(G}Yj8y%{KYHGtTKOjemnw4>o?%5SAoPOv#?2e7*>vf5{1O7s<%CIp?)X!8S zRuKl_CJi{KN!ug1b+os?x<>>}=tYQF_m`e)G3li(Nhr3pZr?Q>(r&V zRg+KoqcUY)0{~EBju8ESHOGFH`B)(@9 z3SYjjEqMQ)XUC3@S6QdlAm0!uK?&z0Hk||ZpSrn7YNW*c4r~E1z{|&{fZ$&PF(<03 zN+Vuo3B|P@Rm0Z?Ko7F%fV^T#>Uu9>-4n`7K-UeLy1M9!f?tqUShsFXPDl`lmsv}R z6G2fi?E8PZ6yo*)M1fXC5t_H?6#&9xIGxhIn zXHh9ob$4fd*}a)u-}5y71j01LV=C(}XFNPS9QqsUMAaJs1?u|7Q|}OIh$8!8`J9*n8Uy#Q-Crt?{En{*RTv-)>8BS#jWDF(EDFGnF|sa&bTu)w9l&V_;K{1$R&Y8X!t7 ztG51=@bEu7Iy)^<`z7_c<*PP@W2=97|NbZDr~=C`QE&w%?hf%;4eYghNW5Y(XE{9* z>|uQ&oyNO#)8v6otC0#vtJIZw(dFQnZ2*;#Xg{JCzW;?wT|yx`M)>KXf{3#gu-SBD zKQ}9iJyr_Tx4rrC-j;Y+1G}GibCN|7kvFktsjP!Z##EQF9VHNqD3w$&!k0LWcGX-b zU_2?O2v%qfluLRzzHOo~k8`u!woYQnZRtB;8pL1*{`o)$s0=pQKY*BAEUJal(HIoh zu%RHZe+&GpV>&nm<@n}QGoOZ~uc3_!!Oi#N6`WCI%k>;Ilzrs7=LoxSeuE~ZLrh#y zZDQOa5$$S8XBf($|6BBTbF==^IP7H{4T!|c!xQt#4yNem?R$x*)0NcIfkLNYi8m-< z_Wq7RMg6#v^_I}9)+Ohg|7vmG1x6GT9D1#5x8xBUIMUWzdb1z~~M)tc5 z#qlgD!kh94O(5vKSVsrH2*dk_HZzST5H z?c;?O7oEn!&dknGyJhXb0WBoe+~bsr!+4n&`SAuxTug$U!2aVNQVGS`65a4vs-c|? z63Z9_aDHq6)Vj0U?n$J<6jD1Tlru_fQ3K8kg*(H}F3is*)e;Q-V*phYab^}4?Yr!v zB*ESm%#z1Fsv%L9xu<$-oLI?2ku4N&IEbQTLtkEE+O;K=$%GPPbBmQW+NW}kI@0d_ zuINSj^#Omk)3Q1O^hB)cTN+r8|ztv6j>dJr}GeJRb16HFB%0*LyQN8Sv0d%cm*WAHO%yPif z1LlyCvVH_v-jbZao+4fcZuV$&R%8K>jiQL(gBp1?Ko4lQR-A6pRfe_a&CII&l$`9` zq?DJ{rDbJ~QcN?XyFQXV9s)9*x!$0!G4laF$aNWm<}6F2f_xzZg#d2=4bZ z>x$dQs1M2`El)qwQ8J?DrTi5M~}4RJ}m-a)K8jJWV5klf$Q-?uSP)nE!Co%fG& z-lr0w*ct8hBw5nbptljZop^slL`Ltl{j!2#_76gR6=;x~mNz?mJL%PTW|Ab}oG0k{ zc(%~Cr>&j$#qE9si{1z;M7LQyCJFE**LGzi%rHUlr4joyksQ(C5Ol!7;*m^rLJN%;B%-5L`o1N&Cm~+`0@q`MK9BwrwNY!6zzh=* z{*n|l!4mwtT|N~RRiJ}a^yC}LbuX_t$Uw|1SE_>`k)|_{L!##rG6ImMDEg9>yg|>G zkr8F1;m(zh)0x7BPZ6f*O9h^BLxUAVFu?vV{&t>c-#+b)2KM(!yB6FB4Wbe*-L~e@ z$XPfq=Qvnm3~7=AlqIhR0$PO}>42g%t5Xa5EXOdF@$hG$@KHtVKB%voV|zzVqli4N zORBh8y(z_2D2c*o>gTtfJUl*5W@i+1Y(vBGqROIK`yL`mLw6wpzh(UvN-JBLTjA}s z3+_Q3426BAZ_|&R-c3I~bOU?Iq0*MoXnWf_vm>Cg40#F83VAxNpxwI(PP8vX5-0Gd~VS2m@fXZC&uyc)mxHn;Xr&{^=Jtx_)J}E4Hde z_bc{`i5^5}SzOa&d_S7Kok{tFdsB83pq4OFqn zIwQM`RC=+bBH1f)nmLw??W=1Dm7#!~FV-qbM^o8M}@^YuU`MIf79wDLf#3$+IL1tDb4c426 zsq&En2jma#cPuIn_)Ff)295Njb=a<^Ur?CGjGV$l&g07M$9|}2%)}l-7dR=m6L94U zNtRT6Z2Yx5P<&YN%P(iqU^NZ*V)~__)2PB}BgafbRzoa-9uz@Rx(@&OEYLe|<2MC= zS9APB0+XvsI2;}awf4~o5hZpkq#+~XH#~4@FdW^zPe2fRU>B}J14uN^lX_7V?>v+t z>@sqocveC^H@(9|iTOyZq|g-orgPC^-gA5Cpo#l= zZO!}t7n@*OrL$yUE$9eu{%#yNgebUc3j_DJI&}om1QiY)dRvhPp;cGPK9*FS!71$L zK2mq@R>A8M#~;BWh$p1Y346U`J^|gNIb$_?9h@sDMSOdAD!*M5pal}?=Z)@3s7|+T z-n^gx?YZ+$wZhWUYKTRo+6}I1iP1y?SBHV@M_EWbRxg3!9sNVxG(_60zLyQ8VG_XL z)0J+f@KFHhN@nE1aN?Vi)TT&NB(Z280%X(1sG$*^aEVoIgK$d;iw%C*9 zqplDNai3v0E0u$?*e-n5e(Hs)Rch zxX-VMxFU1rxJh4V`wN*TV64tR9+|-=s1A;#13zy(r&IeMQp2CNXJJIDGwsY!!8k@7 zOooP`9bq!0OGph`1OJ~J9h)3#GXt(6RAiTgjQ}V!)DgIhlVNW3cjBP%>@rh~c^8~W z@|(b_eBV5ew>XH(G%7Iuapa}+`jx$Z{P72IS!IBhZIN=;0LI-_-6=U%r4B~<*g9uS3k1q9=Ol&4JBX=Y zF}^_P>(jvGfmUMf*l$3TL11ZEkj`QlA%bZFr06)WR>a9G{CoFSj+q_fo7}6d1u27^ z>jTyV?nsU@!$S$63veN=2N86`T&K*!7ub);@oS_sO4?s=%#CJ0J8)v>bk=|P0To#3 z`>Y#dh-ZT|^}^qlnZoLP3qXw3$rbzpULk+(Fs3&XU#2X2r#!#%k3d2!H!SE1ZG3W;NDS5nCnS5 zqkV_Jc6ZpqY_v%mQgZsIwhyJdmwJ+xfL#iW7sppJQ3khp=acSJ2uGQr>c98}p4w#M zF0ZW${SMA4I#uq-k$_1~vtR9-^$f4?Gmnmr-socR&5pb2>m0~~B`VECUPG|c8sEnB z<|^Y2dK1=&d1|7?&H#Qu8n}k;&4CMEn%Pa(wAD&MvpK!z0iypIlXlFADq6V-sYd>; z4Q1}+$1thAeNY`-0PI~f83I~Cgv#v=)8o!yTe)SRtB{=K)nY?6`d+_8Mcphj#TekE z%~`D(aqw17nG^9;al4--txw!EAF1(&C^@**TY|#SBS=TvJ%aT*rW`4)2@Ipt$<-*> z;<4b@G;|%3L&)w+s6~Tcd62YQ78uz+Y+wEzo1bSGwQhk`8^NskvjB0AJr zfjApX9PYrpdH?=>l$Q{hwBQ(wxN@WtR^LjijHVsLFT7yB_bZNaIWHyuCmtmwi1Oya z30wG|RopqvRmOil^C>b-HHUkPD81M-qoB1}25qj5E-a;O5!{!=`=O0JAIcMOm8*@*E92prnbJl{P#J6MUb&#gpo@e2|G)(Y7iq@ zumukBqh^jBvcz{{5=Wl+Fzne&U?`j(UFd93f%JnZ#3PWTPwaQ3)&J1p!&1X_XpzPV zCE+nQ6|2V!KajJ17^g19jIK((a^O;Dd;3p>YkCA$CQ4lz?+d1XG27b3IFymDz|;0h zB>h9n0K+NsT61-)1Dc*KX4+4)jtOO-l9y5LEkKK2cq80Exh0SYqX|yAx_osNRA4aq zOE<9Zpa1+Je5bPfQItJ(mtrBCxa4P2XWqnF2jV&yinE8 z|9Y~V$}#l-413e^Yhg@1>0{V1tv4Mcq@g$pY5sy$IPa*u!=)|N9o9z(LC zmg4Dc{3AFAK^U$F^ecLMdn?kR$|Q2KQ(8~`?Q>(OvaO=H52SjU8v2ycCQlJCh1gj2 zaS2ed))6sc84GttH2}+c4o<}ThE5%<=(*65lBB#jv?f~#=ZR%+*{j4$s4}q}KAx)Kzu|1t zL5BdDS1Tzh4(Q~)ULAKqSC40B=h}0e-QY>CtXOcZA2Ua%j~#rpK@|&I8&y|!bOPhV zGl_hLJ29mPq@cF8R~=KI`yPGO$bYpc_4_}37=ucNJm1eq4p!Dkypj1EPo_|`SE?+> z*so#T;$9aGKa{IKTnPkF0r>w!eUpE97XKd)JDldiWPywb0FD99oe1SyMo~E~ho-ux zn%JI$+z17)Vf1wvXf_~Y&nTpUWT>MWpqXK90HGxFo(&EyCDj2)Cv_{9Q!dHT#(AUH z>s}#|Srg;_09s$U$2>99BDNDOM9vu^ej)_h3R03o4srv5BUL^sIV|h2_Zm%00r(incbaH~#(3nQn5VT-m=z%5A|S7R8<+CpJKRbWu%P?~z+l|&+ChHiiiCg-QKObk&F}VikTdP0cF8lVL=QcgLFPvCF z8V=Y5s0l;C{Yk6@AoAtTbEIj6xHE95Ar5rW3cT9=hLcBb)diV=z`hV&PmA}Nzhv*O zKg@*FQ2`rXqj{ND0nVmPY|a}de1%#Qk{+tkQ!+)! zJbdT<*82Vb`#bjD-}fDRJJzv|wd(c0@AEwOecjh}UgvpUz+s_$1n0u*LlA<`4YMUf z(%_eVJBSNKA>~DkeS9W@kWVOjfli3lG1=p;jM&WSgfWU|`kJv-7-q8aYJZfs6s#Q2 zu3j^UlT5=G3QLd;@IXlV`sLLLTn_$Cgyg^-V&bw&h_sREHO%f(ovZ~3N-pJN42CtJ zTH;|PUqwm=XbjFFt7*)<{1R+(Z%@w`;Fg==Pl#BF&1L42yc$3mLWO~9jUwfHezopl zf0`AjR86PVZ&w7+FK}NZ!hYg;3+PKX_o{7T-Sfy-*yN-`XPFCLp2p2&rY`}h*(T3i zwq8lQY3;W38o8_Q0ZYsG0xWPw%Zez@BF_i?f+MhiMSa3y(vsb!Ku4*&A`JZNqL)tW zD!5nxs|TV0j{u;=1w&=Pi#5ou|09MZ)F$jDZXuzn6P!RY9%Dfa3f` zbSFOUUyp4E-jIsRN-Bgg;cF8M7qVND#+RNpErzFO)FyE@#Qd7OfAVaa-9f*cZtZ#D zGMU7$Gs~*6-}{KDMN!H9{zdv?0Q}xRB;P;2BeSorbAMVJK4)$_MuM=>9nF{IUj^k99|GXD#)?N<)B6KC;Hw^W@{7Twy z%STDoh~O-QXh*~D)SOPdQhC`WJ+D9_GAe8s--A|@{Xh5}>>2l!sRdh>QtASrR38(5 z_lL$ubLSi6x-54p)7s?KBaj}}ce8zerIcwZBrT8Zm;EMRSdTWnRz{0ad%NCy*K}^|F zQc@UOp9BT{%v!^>?!WI^)t%@Jx_H4OGmMxnMqT7fzhm{6f%d0cYVx=1_ciekCw)8L zLip93`2JE)kNaTuwz7J3D^D&DcuOOr)2ZtKVWoL!<|ex&5ns&n^oZ8t@oHrpRt;(i zgM55^Wu-o79u+sN?p2S}`ONIT5&QSs`K}=OQ}8DCEOeP@TM9DQ#Dnp($hgs;t-%(< zksKNtTB5fa?!=eVZo*PCU9Z)+XE055b>%@ECicpiN32nwngk4AZ~;{WXDXw|>1UA$ zSmaUQK++@1!T-1h_szm`W)o&U=^zx$h}-_gN7DLa=n%| zEd6@_F)pp6c5W)$x6&ou?9#2ZWqDCxy8gLU)OCt)at00RDBeXy09J#k|teyOiQH`SgwPFYP zv44_XaIDj#5}k!}5MXOC?|dhM(4))>5N>;%YI zzXx`3y4df6?bAb)u3*utqSxmQ+96(j3`UT<#~KmYV?d&WGyY>}g;Bx6b{k*w#(uxE zb#WWN5wjTo*xmHBE$XM+8EIyMhlY z*|X9=vS*AWNT}w0;1`Xan!|tWC3XlGD<>uA;Dq0eXv z8WQ571H1iSp89Ixyyzn3YO<_&$bxu^mkrr5QK|#S{ylB%`;-4;kSQ^?5!Q+CJGP5) za>@{&(YE(@eS(5^SBreIpM~>@xIUw!fBDZj=TgV>pLx9`J|Fx|Nr{4dP`oERdB(Jr zF_QbQD%3<4AeJQ?eKA4$p}a~tQ{bMw#Aw(1J%5$}!gLuq@Q{|U5Aqz+bAtsNjpd2fe?~&rM=>a?~4?bjoI=>dp z!&sA2Uu39A#*bGxeY))*=}p9H?kEG^Af*HfEl(JBCkr3^lmETE^z!_GCAN(Xe6`>V zRsQsF4Ae}j)}W-Pj{Rz}Ec{C=ITgK(QmsL54vi4Z1D?;%K5qsEI3rJ4Q4y{&XqL%) z<-rw?=C>gkr(8}E3O}C_coq9uy~1Z^@`H^xvaJX#R$G?{F%T)>)GCt;AinkD01JWk<~35sOXL1fWxRt>7=l0M<^A;O z54#*7pU6&08I08OWFbzL&_AMjBAKKa^hMu8nh+aV4>2tlJwhdlX|Nwb*=T;$3h^jy zegzL~4=~pv1d}>;LxkN-Cy@Y!bQv@O;b9@}5=4hE5LSTu+7MV7ZzSp$V{j(hWS|IFUe@g@_O50WG3e z___$e&DA56yjonmY8Tm{{{H@^Ii%f5m;et>%=04IQe91`59pTtd0FPv}X6G+Szc8@4;MlQHJ*l9KQD1*;x!Y>8VyslHXVSYCqx z5q%f_=As!z?z>UJ78AiHnOUKRvlDOtr_mp3PKok3p-#qgM{{~VIiIA0?Jok#9<;W6 zuR^w3n>_x1=r7T1D9)EbDASAEP3vC4EJFpn82m%L3_(PSN1CoZqaULFFW6%;-HQ6y z7Q*mgZmwz+Xcu|@I8(z2AR37~@lMmtwu_ya;TffCv;>+Nf$&U>KD`L(9X9!ZK>q+I zaFqer?Q35K2J;E0atFKr!q^NDT%A5I6@H;=9AVZj!#1K~p;vtF|5Cta86Tu_}tHLSG5SC>7ikvh>f8E3JpPZ z5hK9ZjDrLfw$jh`oW8L%PZ(=p6Al8{dlIpXX3UZStWQ&8q(1QMHI-6MX}XAE~Pbikt)?uX63(n^q+84#G90c%;ZAtxK3`=nQ9iiCDFW z&%(IJn}gJPFhAp}v!B_az67<#n~kh;#pae|F}baG`?(U+V%YB#AptVtp)bCZK>@w{9gHbe@KQ z97fx37am||XD7hFB^XNwU>XLt@P*>;c+ij44uATARI}Af5cR5Uz`zUTS&6)4Q6fC` zKfc7(tc2#hW%T=T1Wpafx;i8%{9{?~x)Eqh<>gv%%|sS1Uxi*I>A*0y_MJ?#jz%gT zg-lA*Ks353r^ycc-V|EGpf332(WJ9RA< z*5X7{NIa3c*w*3TBb24T)y2!yJ0)es9fwa*s!C{TLw`_!jENFo!sVZ+Jv`oYbB+7+(+E#p9SL9E0|XxhF{;c1 z)QIF8e8Q(Ob2Cd>1E*eH9U8fRTQds0@ZSL$mW*Z~eO)la^LrsO@h?P`%2b+pG;`0? zu7y&+)xTq!MYJPaj7r@T71}m7A&@YWCQ(G-Z`%?u;|S#f2HP-HBz>{ZkGCHw5dm>_ zXmvMR*SljU6pG7!kz(Eo+G5Mf(2%7AAjR?Xe&hk`(=*uMZF(Bc-!t)-gDya{?9bNN z6>vBRq&D7dFsDs1mwBH|bNxcnN=mX=ryMNte{K4C7Ety_;2TDW4Ol4E@bxFa*DN7Y zG+SN?+zQPSfExwTNaUb`MzT7J*319&c!(|NxnJ!Cq|>0B{SiYf;D_T?k|Kg}W-9xd zF5djQa+u&qKO0Xo{5K-;uQk>SsSxFI*J|A(mEeCx`@tZLSY;Ev0wHa}De^e7I-5XY zrQ|Z|@>?|JU4& za_>HAH~V%xTfgw!^2Jn*qQ9TsKJ$82(&7CHQ@GoA@A7WjCI#~U{>d-&)gq;%NhSZ2 zbwi<|7cyoHA`>UhS6<<8Vvo6EH6B51`(bB*VibKE%kF?{kMX8Q>W*YBEAsjBXVy;0 z3LZ0*9wAeJLz2{o5@N8P%Lkq!q+sboRg&AdfQSwt*e6|daJ{~uDnWDyLTG@~(<>Z# zHvzCBHeCVoMA(YqH@e!ut&9JeJKl08-UZ6xWtB*+1w|J=6ff?4!_x#=8?sLi_$kBO&o9-Omif6>IXKs1%?EIhFI{*vhD~gVP`4AB|eotjgBVorIj8gESryVhp zPiA10r7MBb6+cX&Fkz;VuHV#oi*bMc=^3^^*bi4naX{dQ>vh?7F%(4T#Z`m9 z5c~Qlh;}4-g-?>s_fw@(o=Ixq{WkDK!HYu}c`wVY24M*ykqII=Qjrl1wW_+5O$tWm z;etjVxj=74a+N{YpCKV3LLN&4z8A=yS6D}81!Xg0VhG`S6Om!mHn4DLV8;Ik`wr^{H|3AdmkW33h!yC9mYRP2mi^k-o3M>zKmKE?c z^^|$TJ$CWcS9`z!A*Ks7uXqs56^qL}-&>BH(Ev5VP$#$(h=IIS&aXSf?Ey|nO|wTH zv)Ej4PypCZ^ETO0)>g4;qm!S^$bC(q5eUHr*U&^ml?8xKk6+Lg<4;Sp)Z;3j&SU zHJKm)7~f%c(D#}D`y)Bj8{!DEQhlR#dObKXH*)$}6)<9)tJGF$IoaQB=F2tIRVzxzWMu%1yvf{`8AXcA z!N5>7$d2Fz)?M^f9n;}U_hxUMKl6adFw~c$RQf&g!Fg)TQ`^dK0X4MeAE9VcuWEmq zt@oXzT^k&dDReDa{T^p7ow$PcDs zwjY5YOh`tEO#6QNkm;Ds9w{22m{qv?TB0 z9;#-GaB);i+*8dxH&;?poOWsWa}$lPJYymYhZfWBAkUlVuCJWOYb8GSpI`EgUfYuN zn#y%*(%koIK+u1?i~|!DRPWupM+zwp!bc@qYjnSP+B>d$l@h5oDRNv_n3#O!=??nt z{_OV>D#jsY@fYy7e-pwouuqvFU`@HiivzY&=mShFVoMx6{4x!r2ku*X#yP$vcCX)* zOf2+ldxO4X;1aP$e9gDYXC=8C4#t_bVBiv|DiM>AqWAnWnLH9L!47!^vmi8$e@Sz{ z@o(t&Qr0m@`;UX{Hziv7?y8pm&aCF*W>d&KeNQfoN&=D?2$}-j!X@VrzEB%uLlds( zdPHO}X#V(#qHZ9*?(iemK)W_BXIO!B)`WPX^XaxRrCiTC=bhp#i{S>vnyRC)%?aTE z#D{5OmFRxQk)2EH*qN3+eF3Zu9y_2@5yb+`4P`v*c5-s|5NbHGaiS+A{t^VXsNtkD zwdI|$qYfAG{6#6@4N_V5{b7r0`(<#n@c&fgoyNfNtvM6fwo!-Dk%}LNs+Ss0v8Aw^;NKs;UW1N=p<>T zq3_UnOff;}MBGN~b>eqSie-fS*8p-L*ONsFT!ZGSa`sQ0mBh@E33h)d>Uz<*)5IZw z`()zxq-_A5qNriahUx9TH|srNY(Seem08A$!cATB&#=KU%na~KfsvX;Yo7kcVuLMd z<_&7#5PNeFn>L!yGN4|RIRyLl|8r&GUrfhgTg~hEiL79 z$+E@q{%Morb%kOwicmrKP}t|gBgVX3ByM(fHHFR4(pNP1w*ZWcV#TdQ!l}7txq1NX zmamF)<71bbogD-rs_JYnM{mh{eZb$#4 zV;rW&#y>zCu0hix8a-kb2PMD|Xyb6`J9PO3anW%de6H`cJzZ0^uhnNtIg$zOerCdj zIk5@^>$LNmyRAuMXV_JMVuNZE)k8cwQCuK)09Qm(PblofbdaM52;diF7_rbu5(z1( zSw`U?sH?w3)h^`*>gIQtCiY1uJ6rm&zuj?4yu9kMIONYH*C$F^AT}ZDr0_^m` znQd{IcLGt(iD(2!qj|Q^U_L2neTd_q>m}2^@V$5r3e2Zm#vCKd(!>+OmpShF;+;Px z!HXPjuc-0J92t_SA&fr>ZuJkQAH((~Y9q+)V6aMTH#p@{0T4nl%x{?@G#vUThrCXV z?>xE^5INVFH4e&L+8RB1weivz*EJSeraf%aM{puyq6w&xNc`c$6)d_l zCG0c7ux*a4P)GpmC%oj>T&r-m|7+MC9jp?0qV>I{OSz~ZBY8Pz%vT~UD8Q~gz{_m5 zw4#XBF@T?h=lN&Ex0Hh$B^0Q~wM4K5W~J_NHr2gjZ*DYnyc5^3iVVm{3yD81S4IpL;lb)I-3 z-Zi<%H`$&+5uTe->tRao9q8xM$ks)JZ5X~4ltk}xmb@*%JAP~!Cnflbm|)wC0l=c? zK1)HM7 zemQp*4gycDiLRGtZw}v#_Pk`XLR4LM9iS5V68a+P9h>Xm~E&TOAO)xP5%^w zqKXXjvZXJ&jdJ!Rk%B;J^Bd!EG6RW=00$H_uf#bQ)WR=_=6Ja4?m@ya^ zzggtmH#r*(H90*>o7)n9>vMs?M}?tqNoV7e{^X43E946rV=i zW7bWZdZ)538v2G4Mvxai`P2S8BRL~eCko*V+{Mknb%^FvaGvJ$ZEf&Y4yMDJJX&&b zxFP0ypK}m-nPoSQ4-R?wZWL@Rbo21=;}A8k|2!~(Ap`ftSKAjbRX4RJ6G~jX|9X}- z7|>57Sl>2=ZzR+(G~?|-yExd=;Y^>O{N$1X5)1p_JeKSzZUW@bt2XR;GtdNJlT#jd z%+f(&(d)NfU=2Xy%5{aF%XEWeszx;tV8S+(jNL_)$pW+~>N4j@q`hQXpZ=BR2%?h0 zy+k=viQZDvPXipOsQ69%^fg=Xpo|Fdf>P!8KFG0#$g)!~=qdfKIT@#ip<on_of5*7?&6Ox z`EPl8__M#ZS^ot~OQrgF2EN>@!e@6dZQzwN!uzS{sHrxTib>X=I20EvhC&7U3TQZK zh)khQEETCha!2f~@vJ1KeuOPTy+Q>0wi=UmV-wAO7G+2a_cGT{L+REDaA)8@ zm7@(sCYhi~95#rD8u;u_vTXYzQLn#3qmSrQNjCsO|$} zGy6=f+iI#(cJ|ILZ~RvW+%6D%?${Dv=a9SF^ekqu9PF?U5^nV$U=p z#|FD4o6TltX9c0sDhra>jHxK;(mbm@*2pHTbG*pHz~I>|Us`sg`?g>S){MLKV~^}r z!qHnMX1#wJN1u|f#ZBMwS~qnClnoycOD=(TcM@aXfcy-Nj~@jT1cQvFHa%5~m6fA^ zPwfOu5UTq@UvDn@65evdh@lameR%9cA$Inoritu2g$)fGq4tqRzluMX2QOxiHnADs z**E^MX-08H=#tyka$M}u45!|4lE1qfk-bkR_-BhTd+0Mm&tvK*ZfB-dxZ*ZYcvF67 zf2Q+&BDf`Q+5Mv;TP`f_4>;Y&hreBRXME>6h8phz_fEJ1pIs>X$3xD4vx?8}e6k6(!0akS0IjlK0gk0C;`?3ho&kyyjbcTaX*;vzT3< zFNv?s*Bm@T=T1-AH}de}`w6kfCv3=L)75)oE{XX4udI7yLxj9MuS1*>4HanhsBiDKM0A+{~EM|9iCaJKl zfTe5E?hTt)@bJd3-FLY(-Usv;}TcY&-+HJzE_@etqgR?7h>4>a>}O@@|OivMew0`yL0T*Z(fAp2KW2onOv4u z7}WaCWWp{a`idC&2< z3T4Nr4W3)7+!ikpbVlBmfa;LA1lYsD_seR7KG>o!p&omC8+ROw3Mh-|x_e{AHCB7a z*q+IIxI8ml(<-(`A0MrADbuO3`>oThoDW4D^zYT8IZ-^=B)S6J)6LM4-$EkW0bw|i z4<5%6t>8BZPlrZs2Aw$c%=r(R=9zueh)K?!IDA z62No9EUFu_gR+m2WSl{1tD&nIRv8i%B{$&h#`nG^d9?RekHGxY1N}IBw|-5H>@#{# zET2ZZFP30vc+@K+L}`aC-RJ4FDJR9zliX@Mi?=&07(}_gM0V`b>K_aA(TbWnV4p91 zy!_QEj(C{64m_5AWK^e^BjAr~t=QP-ZlUO#q0;C%cx7jjXwCgjw0vJ}@hWl_%l4gr zKF*0nb)56{B`Xp9eKdy3%kPZVuo>s<Tt-xz=V}R%>t2$N(dq5dUYN=RcH9 zJ-3I*={tVN3oOm0-T6Gv8m^O70nN>3M^w^^OHSVqrU4I zp1s){#&OxreI@0Vb<#B_I@l*sKB~9$d04!jcTl<*V2n8|WC6PM--_cMe|d?!#FX`{ zd(>64rsl@t;^mtjel#>UYVHX0*jeLXL!Pt++T5P zp?sunF&<;|rI`a8)|WW!)8QcdRHNsoQp=2@`{5Ukvzn4Q_fs$0+MP)}X1wUGt2LF= z{r>v;PD-q0w+1|f z+kSp5j!sUMa9rcl&sq-?^K_9$`WsTZeNs}=MGW6M z7Oe=JZ3x|gGbl6U z+W94aNX@2WdHSW;~}Hi&@T>wiol)(M@*MEnQf;u zAd-L)o0Jri?n+c1H+yg}NXyABqYP2B*0{$$H#l@B4{Y_pq2b|wE?p9YFXGAOF9=F* zZhW%)#Jwe3o*x@!!6!rLti|n&YEOk@=gxN2&IK5w^{uDp)%tQcRUAYCcym{f{Q}r4 zX29%dkI~1K8rFDbZ$W>jC@cz*10J$rZ}B^@(M0j>V-$SwigG5SB~Ka_H?BH*%X9Lf zMO8NATzwtLPh4L8+Fz_;i?XZ(DPJLy@mryx>>mmWw(s1z1h;nrV3(@4_A+X}ZSQ*J zV9AH$E!nfDtt)~g?t6JvxpczjYG-$~lA@B5%+;$0kkRry#88a$H@lPy;4rfW{kRh~ zDLu-5!RPx{l24{6`l;5N-PtV<^9Hxf^c{Ir@z0_+D`c!QeKIpbz=rtIl6#Qt>W}Sc zo$Q-{IP)HodyTutvaGw@hx1T@;KPFL@N9UPifbMQ`upF*NCtZc2aQd7`&v%Qkb4AJ zOtWX$xSQuuAc^V5N3JZ~-!oU&x^HCKdzFd2dbUoiT;9dD8`g`_Gh;@H4_L9VqN1W? zVS**`K^<2lUDua3Iww@I91@HA4* zm79aV@gvmfhc|B4$>*{J{-U;iyr+68o~>Ct6vzVuu{ocLiv`7k=R{^rqSt*@KW*@& z-e&2hVYbg!bgCx}`;L*XQ^K%JXk_kcXsU-*S2`PG%@V{Kyz6$6O{=6AK}GZfmeFb( z_CbC+P_vQPhIP$?FINehtgsRNJjaF~LN)g+qUCA}#(w=et<~39QM1mqCvFmVPC6}w z?pq&8-TFOxk4W{L6WNqvRF=bsbwnPfYcUz}=*7+%wusFGWWIm?G)pIPW zbN6aB?rqt$NjD;cycM3qHGDB#`)9M%wG6BWY?cAYJ0dN;2=t2kh%SbJd$Gj48)Nt} zpKTp59gxmo{UzJYz|h~E%|d;~`O8y?w*F%HgPQLh16}4K;+I(=Ii2wHRV`Oe(tcZ&foNLXKz-_=x%0$ee$w4nN162gW`t`TS9zqa54C-d%c|*tyDOVW5;spXr zs`Dq>=}+d^Gf;aljF_je=A1~$3_LOsD81f_lRJy|YwPcS0PMDI8!gTuLs|SVN=zzo zIkImzRFgqAPAGBm89$$+yd}R7nkzz(4^JN-4-e7FTCC zyw=ZVhO5=l!-wg1ssuei6SiqA?MINMMQ=|X?dI2h7<_57D_OzwWF;QH&y7hO0tGOW zQG8N)PW9x;lf_jUS9+>AM`1*p+1S_9bI;3*5f+7X6vPuASmp3ggXe?~Q!(}U@nct% zyB#^m4Pa529YY zgR@OXXv^IX2EDZX8#FY{-CV4`bysFa#s=ap>)wvfgaZ&Y5wmRxg+j!pzn%e63 ze*YtHJned+pl>?#>(}vXzyG{QS~T*iXMm4SM9vyC zXx-c45Bv?1Kh0~lfB!!Ec&;3qm^BAX(GW1skG}f+_bAw?V zWA6`G=$l*hC3dIPnx$e+_M_P9ZAr_Ri&uw$ghRQ)u41#PbvLxOtEjZp)P!8}3lioR z`p#U_WBK~^YJIJA+S_Yhv?CHhE)VZ8DdE6UNhhR&1b2(F+I&OiWFMqB*0TWy1~mx+SljuI?r>vmwR2fsu`k zZD?eK#5f$=3mJ}|r)Ry(**FEjj99Slf!*cGjY5l1_us|($EgkTlrPQHQloogd5)7p zny*8~7?~!PJ#jS6`0-DIQ?Rgn>Sa@R|XT ztrGk~K7%|~xK!|1-7`w)bjS_`xW(P=4QNHf=->*)lyQ(GyzNBCvv3xc=oz*0IhntV zjBMXJKd}^hkK{c_cty@vEn2u_ep#dM>Gz^*~Y&Nc5YmtCu6g8 zU6MP{O?jZh)LN>&r-$3p(z2u#uFrm>qobP&9(j1MfN{j>>*wc}k&(gWGL9)$=!Iys zuuq%3N;?H-5yr6g3ji&8t=w)VNMD20)=f5yyY6F9P*8hY+wq^tFbGvVe%$kRTG^a- zZ#|p1{l!eXSgb0Da9%c{NL}|1iU{p*Uhr!EY*K?`5 z%U7%rngpv?(nPBjfboT9v(6X~y^TicJ?f8Qw^=G|~!NbVg39RRRQ9rnz8;%@-|@hg4v~eX7tR(nrj&t zwo~C(23urb`gwcH0oA1*!$(?^*A8#D*i{_pr|e5p6O(&bs?=k6-KZbDa@43eZEs{6 zOL48Gl@kzQjmAUJ8P?Q~75Q4s!pDinVhh^#X6DySV#lBz4JJ2{%!b8>r>4oJ8DC45 z+x^MQVix*T?&iCS3JPE3>e?E5n{q)=@5*t4&uC3MV-2(8MZeo{v;7829pwr#@ut9| zz?MM9rCiBOr;mq$0}Yv_kH%v9Xp$+0H<5Z+u`qLg5e;p zaA}JV=htT|8?thN)Wa*^))w9j{Ob2Fc$in#;$}>s)O(0k&<>ZF1=Q$|%7x&*=EP<^ zdZRlq3nlJ1keUfhXZG7q`$Fm!H8nx?V<9#Urt)^6Dx+Ls zaT0LAb_MFke-J_1zJI?12E}RETqsm(Z?5m1F9t`S9jQJwR3`kX$xb}9(M=}lcF){< z$`v8^S%8YmvpFHNj}>KQ(sp)sK4%WHE2fEVOftDfJ^>zwxI-HR7a*gL{*bCwYt{$y zok_-86%`dA`U^0kuISUJLZlwQfVpcOjNFkVdkxw(R?OO5jlTa7Ir|S4A9PE7Gzfsd z4VO4HG$iHhEQyNL1MRMD03$(}h_e$vYFwNq=`xoyJcktkw}>6YQ@AO|VI3ZB22KR@ zBmm$n%R#F{w5a75xiV>Gb!G>o9GiqqE2Obr0cg;9 zw?tOP8ysSlbldYfHf z=PSpjgQVwu7x!u*we#3h!wz#dR+jt5^dne*^d2F7;p-u)2> zbs{w7I2{d<0{*CrH+a|~#;C{ob<@KoV8pxwbvgmz0y)_koE;NbYgW8*I}(hqVdpJD zQ*mcbPEG=JSLW8g8cMVV!9ak>n%jwq%2?=-f+BcXxIiehqad7$7E>qLqxI-~5b}h8jCQFaFoq8wy%GW?vrH z3V#lN?OdML@t5=AAEH0h&U^GI|M~i6R?SSE$N?gx*aXi=mwVZ%j$JhG%cBf3E*`GQ z7%TD`2n+~VN!f#T)iy9-G1QpjAWUt8TxYT$jE?SM<>lX?&@Qzoo6})nGS4_8XuEERagg@<>A&=4ou7 zk_|7fT&+{5#UR;hEG#TWSm_gJFZwKS0JTzglnV-dZ5`&Q%8pxbLF^S-N7DD*ai_>>!~Eu9+l zxFA+1S(ZY=T<*5g)U17;uE@USk#O-6*U`I7pqz5Yin!wh$H@KoQc;?4AphOqYGBFL6)?O zf~sI=bb-wxU$a3}oypZB!^7cldsrYvEI3h6*C;i&X%E+v(HM*ew{L%IcoiTG!hs4^ z1QCFChs8yX+5>(-6y)wDqqO)c@9!pkOD2g4TSWo8bg?T+ZB?=5%q%RWB*7^xJW3UT z#l6ZG_6Bj*Rl=H!S~D&knOw6=L_{4UnPj9nKqyD{y&VwZ0x8tf=9LyHEWYL!7(782 zNp4cyH%}&S{~W=R*Rd8JxD(Nsjg|wkqFi5uKTG}iyFWo|-kru3Dc zs6d{eDXeI|;&|*k_67xLeK_HkDc0a{^{RVzcJ}ZUP>DDtB$^E~qXV{ebaWJeWAFb7 zscedF?C_O*=UKCw<<(vmvCmI6T);#3ZI(ND=H3{PmJlJOm>LC$(HpcZGE2{XI{ zfQfXV8TeG#bT#sXkjit2+zbFjKt;srTaEo6d%PdX0nV2;B(Tx|oQ`X2yMZ1@!u((FC0_nGATZFUeF70f3T`wHh1uDy`N)fRlQSM$U?G(= z)_7rW#T~aF0&pC4b=$Ab@*DLxk*JXb59hCW;A#fXBLsZ0Yr_e@m_j398$0{Ej}2=X z#P;Gk;ZVvb)gw_teAZ~6z)C*h`)-502Z%KbC^*y6Ka@4BaFjb3nLGBcDFkk|Tzwxll_>3juNsklMG9OhK<}h9*V1VWzuE)GT@@d$8$98@HvU z1<2K5G+#-U$}S)fuaF5}J|h^l0AG>9=bqHpXIQ(|m7s2rFcp=Ry;ch#R0fh%!70_*NoSp1=m?)t|%T zFZm@1kx=r8rnoOp^u+ouY&yrGw@ZICrt-p$`&(Pvb&$QRR9g9lcbaQaBfg}T+KE8hbsaYxTA zSuk&ASdml{IE{x!M{l7i1M&H7(5F`KQVDu(_w`3ie@`vbOJs>$^UQcfbc{?2{2D9EPx?Q~LgT9u_^RZ->GCSWm%&ah;g+OvaaZgVxSrf?SYl z)8XjRqxU^LJQ~DN&U9iuo$bg0jNbnJ``Nw9h^bvDX?7Zghlk4{aaG#gicDM#qrZS0 zNmFv*ldL+!v1=Fa2#VJ_^NOiLBHnsCii&}!S|worc5IBj{?GCD{jK?}{ZrBGaC1Oo zjV*cLM|ZcWjXu&ln0l~N#=!c0;x1qnDwN@7G#XxoIOf6TeQ{MCHjwIDqmBsc`+Vrs zIz~pKg=1Ku`qAsA@Mwp&pZ_9x-O$<1QYxjD3oOIFa1-=7P2mHcBRpq<5B#*?M%uG7T zkPQH*A>Sn*08RKK`U9~83 zj70p9I=ah-9nX7>lF|UItKERKaFXo6eagb#qo;n2jrpRw+3;`#RW?^2{qp6Ws8F=* zel~G0U%osE69N4(0i#Ni(7;jM-rOK2Cf1ebt=04r%bo#%IE3r37A;w#n=4nlFv>yLoZ+7vvy9vs7eymRaI3jbrBvi-^Jr(PC~q*-4PCsA~L<|=TagAmq+IF_JB=c z3`{V=Ka{}w+bNJUbOX&HEqs$s4EFORqjYo!!oNFj6t_(~ z^!M{YF6K)NWidpG9TzkVbW9N>WN%wimdlQ)#t?0tH}MWda+MTMJ`|@*72sG-4}gW6 z9_|Qj$&~3v9Mr1K#(cQs5k`jN+$8Y3IiFrAMMk`b8B}1@DL2-PLL27B87Br19bV@; zCMFZR761Tr+sOFQ;utIT2(1A0DpyI?IeFI1QVC$thLz%rsre5A#l8DhoJHW&dOR4q zZN#^<_@615w>&|}eq!KG92inY@GS%|wMVuChoCk>)?!yQq23x!LG0`82s)uok7TNy z=SnGJ0~mG*Iq{1!^dL^#1%Cg*vW=uJl@3G$Q&x>BcsK0;i9Z)|LLYBu9i-%fCXz;4 z)~y(<)U^R>SWoxVl32-Up3oN)7-32yb_-zMk|!hTF=cmBugA3M)q)cpo)Mn zZ~|suGV&vC<_ad+e=A(#1a~?wfB*AC4#;_ZND2Hz){mcy>oJkUPG0{|uW>Tf-I!|c z>f%IX6n^?LAwjiAbfFaJ115Vsbh~}s1#SBRM9zCPINWoKT7Ta-R98Ab0gkx)^v)?i z-E-JKJxePl+6dvS&g6LCly>gpq|RhC0Yi?ZD5x0*!P#cjAC+%BYS)DV)3c0ldvsV$gP@HF6i zM@N{rR6n>ba>tJG?VE*uVF87+Y5l2Fr_R<@V^eg^BIbFIBiWk({eaH9i9a4x<5@_LJ%ZSAM?E0kNWWRb5PgE(hI|c{< z&Jz)9Mn&pi@T=#GS)ks7%2|pi#>}kNF&ww}fRoo3-$m%4eoPII!RrT3nHu-;Gn|)n z+7AW!J$%`Bxb4_@6A--!LV(cZGCv!om%babD{kPJR@dD#Jvr&47cYpe@`;u{t9kfB zowgIZJnFJmmG{xlXin8!Co?6VE|cik7=`>*qedG$fURWz%qle$3q-zn>-KFr3bDfc zB45u6Q=l~u*d!Va9a6_)@=Bb=qhEb(7JwBS8|kB?Gt-V@#V77l{TP`D{uHBvNkbL% z1R#6F_#9n*86dM56bi)B&O4hu_C$5s-kJ(C(m!qJQC4Rr^I>XXmXeEe=bG_Yit$44XIK2mS=uXJb)O(Hk89*I{z9 z0A26;Jf~}!TODjH^Z^^rKtrcd(afC79sN)2Wks2Sf`XEXZ&TfR;Ql_85jxJV(J%z1sRymrzGuU_F{Yeb#pDmEcr?wu!!eOtpnDc!F3?Sgh)L zT0YL3K}UPWe+*ET_l?6YVoqeS!`l1~LA2r9!0mMi&^`dF1$$a*HgV+}I5|3+~abU@6c0AfcFHl>JGFWp5kD#-YTTp zSc~N5h^8hpsER#9TJkjq($#IyS?gN5Vs?a|fx-Xd&synJb>M^zJ%t`y30H*r27D%2 zn{CQq(S@l!ab#B#bk@;PoXj_)BF1N)xofGa8jf!NXUci9i=``fZVW>rxwMj3Z_+ z@(6%Q^`7r$X8PE2(0$I*c}uFozhvPX1N~S+h&L`;8&--Uig(b;G+*~UYA}2&?%?OV z-$w^4<9JXsLr_9iBy!ht^$FXlZ^-?)$2%Ux#ol*!=V=~%Bt_N)@wgp6H7|Mg-!+j6 zvfj<5g+KIA@eH5R+N1qWV!SELMmY9$QOtvwm9j zhpSzR{GL2nN2WVa5op|J zm?Fhz$uzL9rrN9h*Gb0XJU40DMQE!J8uCcT&OYD4yn6#y5$zv zs+i4dBt{?0T&`r|1#Tm%r%swK$hUUB1gT~_8fQ{HHBo>up9NgODN!;TF$Lsd4`ON8 z+z-V3&FiHBf~oz?bc8Oo)M>fscbxaK_iz72d!zx6C0J`YK9~ek?ml9m@AR>l8j`_L z3h{HACc)3brajxx{=lrGtt%s0(Aj~RM$Oft&kf0@49IzH8j<7Of$(mOF~0He3zy#uI$XPEgj|Pm z1vp@c4wnye368S@02}2MM$6Na4Q!fI2(a-y=geHKo>d#{;Dj^h;NRp3_0U=R-}tYt zLSY~z)@wH32R4FyjgEp?_5;k3%1$})b92A-GjP_4J+-zrvNlsDIId6%HN|IC-L{OJ z>{-M@=VZ{Pig z4@KRCulziD)5rog$HVA-DS}B2{;wXHw^r8C*0vpNg`MM0Ca68PAw^Zl0b@fuZ*J-W zcmC^#Khw1taBWBJ%gHKpC~1I*%4XL@7E@#pp_Un}G6RjEfltz9)~* zm;s(iuG3%if-g?`<{v@I6nRWH_h)U0oYPlN6NpZZKbCs;k#09L zgC}T#C~sIObYt8UF7IFuRU-)}!z}wGw6uQY{`?CNO?Iw}>C{kH=fIazci(*CSeNcE zGHMG}>5*@Trwojd@4s+7zjq8~1n_kZe_DPGvBMhVOf7CQDeK841j=0i6E)n0vHQXV z)zs9&aSL|(U4WO}3=%INW>Bs{s@}H|jD{lcj!L#Df=sweIp8tU?57WUP$!gl2x8e` z9LvoW0(UJQ5xJOyw}7I9xO%GZoKw=kh?L%?+Si0ELf-&Efe1I zhi+~PiE=xmWq6qPLK2+>B6klsvw%C?V`5?mjD;rdHgcM1T-HlD=Sf9qP0oR!9X5h8 zSr+sx2?y}R$?Pw}_`#-QK)38XhH4-a)CH5%!g>o+eo^>|D55QP5 za3v=ApqQ0LD+k1TlZl}=0)8+E0+y{3_=qqO5fUN=uejaBcoFgk@=;A#s>3kwDFT)A8i*e()cOHl>#AOiX# z73;EJ1uP0&mlDXWIgx_x&3YYi-x$RNpbB0P&(@->sx(VwT)&RsPfgm}1_B^C)525vnjz9sx~wk8!tj6t+I=)jLHcq!WN& z3=IjP0NpM}^fKy$e2{es>T7Acw{>+aaJ}E;TD7YN9@7o`cd)Y;rs6(=ni>LVdjnkH zXk892(pIn(hf06|{>bX@V7E0K*aBf5k)De$cOe9@r~P0)itbKRc6F00EH7UL9E?-| z;FVkS+2BLf6cko~4|W@wNn?$LsI>H>8(@<*_@JnJfmofUeABJyG zeSg-9#=Ha{bEv15Fe~w6V$b>iO33^_Y9P_5oBu0{=l|jz=hrk03Eq6!zMvNZQ%YV& LSvu|?!yEqtBfMxE diff --git a/examples/win_at_p.py b/examples/win_at_p.py deleted file mode 100644 index a65fda87..00000000 --- a/examples/win_at_p.py +++ /dev/null @@ -1,552 +0,0 @@ -""" -Compute win@p curves for authors (agents) from Trace JSONL files. - -Definition (per request): -- For each workload group (same definition + workload.uuid), identify a baseline latency: - * Prefer runs whose AUTHOR is 'flashinfer' (case-insensitive). If none, use the fastest run in that group. -- For an author A with latency L_A and baseline latency L_B in the same group, define r = L_B / L_A. -- win@p for author A is the fraction of groups where r > p. -- We pre-calc all possible p's as the union of all r's across all authors (i.e., p-grid). - -Inputs: -- One or more JSONL Trace files. Each line is a Trace object, matching your schema. -- Optional solution->author map JSON file if author is not embedded in the 'solution' string. - -Outputs: -- CSV with columns: author,p,win_ratio,n_total,n_wins - (n_total is the number of workload groups where this author had a valid run; n_wins counts those with r>p) - -Examples: - python win_at_p.py traces/*/*.jsonl -o win_at_p.csv - python win_at_p.py traces/*/*.jsonl -o win_at_p.csv --baseline-author torch - python win_at_p.py traces/*/*.jsonl -o win_at_p.csv --author-map author_map.json - # author_map.json example: - # { "rmsnorm_triton_v1": "alice", "my_fast_impl": "bob" } - -Notes: -- If multiple runs exist for the same author within a group, we take the MIN latency for that author in that group. -- By default, the baseline author ('flashinfer') is EXCLUDED from output curves; use --include-baseline to include it. - -""" - -import argparse -import csv -import glob -import io -import json -import os -import sys -from collections import defaultdict -from typing import Dict, Iterable, List, Optional, Set, Tuple - -import matplotlib - -# ---------- Parsing helpers ---------- - - -def load_author_map(path: Optional[str]) -> Dict[str, str]: - if not path: - return {} - with open(path, "r", encoding="utf-8") as f: - m = json.load(f) - if not isinstance(m, dict): - raise ValueError("author map must be a JSON object of {solution_name: author}") - return {str(k): str(v) for k, v in m.items()} - - -def infer_author_from_solution(solution: str) -> str: - """ - Heuristics to extract an 'author' from solution string. - If none match, return the solution itself (treat solution as author). - """ - s = str(solution) - if "_" in s: - return s.split("_", 1)[0] - return s - - -def get_author(solution: str, sol2author: Dict[str, str]) -> str: - return sol2author.get(solution) or infer_author_from_solution(solution) - - -# ---------- Data model ---------- - - -class Run: - __slots__ = ("definition", "workload_uuid", "solution", "author", "latency_ms") - - def __init__( - self, definition: str, workload_uuid: str, solution: str, author: str, latency_ms: float - ): - self.definition = definition - self.workload_uuid = workload_uuid - self.solution = solution - self.author = author - self.latency_ms = float(latency_ms) if latency_ms is not None else None - - -GroupKey = Tuple[str, str] # (definition, workload_uuid) - -# ---------- I/O ---------- - - -def iter_trace_lines(paths: Iterable[str]) -> Iterable[dict]: - for p in paths: - with io.open(p, "r", encoding="utf-8") as f: - for ln, raw in enumerate(f, start=1): - raw = raw.strip() - if not raw: - continue - try: - obj = json.loads(raw) - except Exception as e: - raise RuntimeError(f"Failed to parse JSON in {p}:{ln}: {e}") - yield obj - - -def collect_runs(paths: Iterable[str], author_map: Dict[str, str]) -> List[Run]: - runs: List[Run] = [] - for obj in iter_trace_lines(paths): - try: - definition = obj["definition"] - solution = obj["solution"] - wl = obj["workload"] - uuid = wl["uuid"] - evalo = obj["evaluation"] - status = evalo["status"] - perf = evalo.get("performance") - except KeyError as e: - # Skip malformed records - sys.stderr.write(f"WARNING: skipping record missing key {e}\n") - continue - - latency = None - if status == "PASSED" and perf and perf.get("latency_ms") is not None: - latency = perf["latency_ms"] - - author = get_author(solution, author_map) - runs.append(Run(definition, uuid, solution, author, latency)) - return runs - - -# ---------- Grouping & baseline ---------- - - -def group_runs_by_workload(runs: List[Run]) -> Dict[GroupKey, List[Run]]: - groups: Dict[GroupKey, List[Run]] = defaultdict(list) - for r in runs: - groups[(r.definition, r.workload_uuid)].append(r) - return groups - - -def select_min_latency_per_author(runs: List[Run]) -> Dict[str, Run]: - """Within a group, keep the best (min latency) run per author.""" - best: Dict[str, Run] = {} - for r in runs: - if r.latency_ms is None: - continue - cur = best.get(r.author) - if cur is None or r.latency_ms < cur.latency_ms: - best[r.author] = r - return best - - -def choose_baseline(best_by_author: Dict[str, Run], baseline_author: str) -> Tuple[str, float]: - """ - Returns (baseline_author_effective, baseline_latency). - Prefer provided baseline_author if present; else fall back to fastest run. - """ - # case-insensitive lookup for baseline author - bl_key = None - baseline_author_ci = baseline_author.lower() - for a in best_by_author: - if a.lower() == baseline_author_ci: - bl_key = a - break - - if bl_key is not None: - return (bl_key, best_by_author[bl_key].latency_ms) - - # Fallback to fastest author in this group - fastest_author = min(best_by_author.items(), key=lambda kv: kv[1].latency_ms)[0] - return (fastest_author, best_by_author[fastest_author].latency_ms) - - -# ---------- win@p computation ---------- - - -def compute_ratios_by_author( - groups: Dict[GroupKey, List[Run]], baseline_author: str -) -> Tuple[Dict[str, List[float]], Dict[str, int]]: - """ - For each author, compute list of r = baseline_latency / author_latency - over all groups where BOTH baseline and author exist (after baseline fallback). - Returns: - ratios_by_author: author -> list of r - totals_by_author: author -> #groups where author had a run (and a baseline existed) - """ - ratios_by_author: Dict[str, List[float]] = defaultdict(list) - totals_by_author: Dict[str, int] = defaultdict(int) - - for _gk, runs in groups.items(): - best_by_author = select_min_latency_per_author(runs) - if not best_by_author: - continue - - bl_author_eff, bl_lat = choose_baseline(best_by_author, baseline_author) - - for author in {r.author for r in runs}: # all authors who attempted - if author == bl_author_eff: - # skip the baseline author itself here - continue - totals_by_author[author] += 1 - - run = best_by_author.get(author) - if run is None or bl_lat <= 0 or run.latency_ms is None or run.latency_ms <= 0: - # author had no valid PASSED latency → always lose (no r recorded) - continue - r = bl_lat / run.latency_ms - ratios_by_author[author].append(r) - - return ratios_by_author, totals_by_author - - -def build_p_grid(ratios_by_author: Dict[str, List[float]]) -> List[float]: - """Union of all r's across authors, sorted ascending, unique.""" - s: Set[float] = set() - for arr in ratios_by_author.values(): - s.update(arr) - return sorted(s) - - -def win_curve_for_author(r_values: List[float], p_grid: List[float]) -> List[Tuple[float, int]]: - """ - Given r-values for an author and the global p-grid, return list of (p, wins_at_p), - where wins_at_p counts r > p (strict). - Efficiently computed by sorting r-values once. - """ - if not r_values: - return [(p, 0) for p in p_grid] - r_sorted = sorted(r_values) - n = len(r_sorted) - out: List[Tuple[float, int]] = [] - # Two-pointer sweep: for each p ascending, count how many r > p - idx = 0 # first index with r > p will move forward as p increases - # But since we need r > p (strict), we can binary search each p; do linear pass for simplicity. - for p in p_grid: - # advance idx while r_sorted[idx] <= p - while idx < n and r_sorted[idx] <= p: - idx += 1 - wins = n - idx - out.append((p, wins)) - return out - - -# ---------- Baseline inclusion (optional) ---------- - - -def add_baseline_author_curve( - groups: Dict[GroupKey, List[Run]], - baseline_author: str, - ratios_by_author: Dict[str, List[float]], - totals_by_author: Dict[str, int], - include_name: str, -) -> None: - """ - Optionally include an explicit curve for the baseline author itself. - For baseline vs baseline, r = 1 for every group where a baseline existed. - """ - count = 0 - for _gk, runs in groups.items(): - best_by_author = select_min_latency_per_author(runs) - if not best_by_author: - continue - bl_author_eff, _bl_lat = choose_baseline(best_by_author, baseline_author) - if bl_author_eff in best_by_author: - count += 1 - totals_by_author[include_name] = count - ratios_by_author[include_name] = [1.0] * count # r = 1 each group - - -# ---------- CSV output ---------- - - -def write_curves_csv( - out_path: str, - p_grid: List[float], - ratios_by_author: Dict[str, List[float]], - totals_by_author: Dict[str, int], - authors_order: Optional[List[str]] = None, -) -> None: - if authors_order is None: - authors_order = sorted(ratios_by_author.keys()) - - with open(out_path, "w", newline="", encoding="utf-8") as f: - w = csv.writer(f) - w.writerow(["author", "p", "win_ratio", "n_total", "n_wins"]) - for author in authors_order: - rvals = ratios_by_author.get(author, []) - total = totals_by_author.get(author, 0) - curve_counts = win_curve_for_author(rvals, p_grid) - for p, wins in curve_counts: - win_ratio = (wins / total) if total > 0 else 0.0 - w.writerow([author, f"{p:.9g}", f"{win_ratio:.9g}", total, wins]) - - -def win_curve_ratio_for_author( - r_values: List[float], p_grid: List[float], total: int -) -> List[Tuple[float, float, int]]: - """ - Returns (p, win_ratio, n_wins) for each p in p_grid for a single author. - """ - # total = len(r_values) - counts = win_curve_for_author(r_values, p_grid) # (p, n_wins) - out: List[Tuple[float, float, int]] = [] - for p, n_wins in counts: - win_ratio = (n_wins / total) if total > 0 else 0.0 - # print(f"p={p}, n_wins={n_wins}, total={total}, win_ratio={win_ratio}") - out.append((p, win_ratio, n_wins)) - return out - - -def select_authors_for_plot( - ratios_by_author: Dict[str, List[float]], - totals_by_author: Dict[str, int], - authors_order: List[str], - include: Optional[List[str]], - top_k: Optional[int], - min_groups: int, -) -> List[str]: - """ - Choose which authors to display on the plot. - Priority: - 1) if include is provided, use that (filtered by availability & min_groups) - 2) else take top_k authors from authors_order (which is already performance-sorted) - """ - # filter by min_groups - eligible = [a for a in authors_order if totals_by_author.get(a, 0) >= min_groups] - if include: - wanted = [] - include_set = set(x.strip() for x in include if x and x.strip()) - for a in eligible: - if a in include_set or a.lower() in include_set: - wanted.append(a) - return wanted - if top_k is not None and top_k > 0: - return eligible[:top_k] - return eligible # show all eligible by default - - -def make_win_at_p_plot( - out_path: Optional[str], - p_grid: List[float], - ratios_by_author: Dict[str, List[float]], - totals_by_author: Dict[str, int], - authors_to_plot: List[str], - title: Optional[str] = None, - xmax: Optional[float] = None, - legend_outside: bool = True, -) -> None: - """ - Render a single Win@p plot. If out_path is provided, saves there; otherwise shows interactively. - """ - import matplotlib.pyplot as plt # lazy import - - if not p_grid or not authors_to_plot: - raise RuntimeError("Nothing to plot: empty p-grid or no authors selected.") - - fig, ax = plt.subplots(figsize=(8, 5)) - - # Plot a staircase curve for each author - # We keep the default style and colors (no explicit colors). - for author in authors_to_plot: - rvals = ratios_by_author.get(author, []) - curve = win_curve_ratio_for_author( - rvals, p_grid, totals_by_author.get(author, 0) - ) # (p, ratio, n_wins) - xs = [p for (p, _, _) in curve] - ys = [ratio for (_, ratio, _) in curve] - # step plot conveys the strict threshold r > p nicely - ax.step(xs, ys, where="post", label=f"{author}") - - ax.set_xlabel("p (speedup over baseline)", fontsize=14) - ax.set_ylabel("win@p (fraction of workloads with r > p)", fontsize=14) - ax.tick_params(axis="both", which="major", labelsize=12) - # title = "Win@p Curves by Author" if title is None else title - # ax.set_title(title) - # ax.set_ylim(0.0, 1.0) - if xmax is not None: - ax.set_xlim(0.0, xmax) - else: - # small headroom on the right - ax.set_xlim(0.0, max(p_grid) * 1.02 if p_grid else 1.0) - - if legend_outside: - ax.legend(loc="center left", bbox_to_anchor=(1.0, 0.5), fontsize=12) - fig.tight_layout(rect=[0, 0, 0.82, 1]) - else: - ax.legend() - fig.tight_layout() - - if out_path: - fig.savefig(out_path, dpi=150, bbox_inches="tight") - else: - plt.show() - - -def main(): - ap = argparse.ArgumentParser( - description="Compute win@p curves for authors from Trace JSONL files." - ) - ap.add_argument("inputs", nargs="+", help="Input JSONL files or globs (e.g., logs/*.jsonl)") - ap.add_argument("-o", "--output", required=True, help="Output CSV path for win@p curves.") - ap.add_argument( - "--author-map", default=None, help="Optional JSON file mapping solution->author." - ) - ap.add_argument( - "--baseline-author", - default="flashinfer", - help="Baseline author name to prefer when present (default: flashinfer).", - ) - ap.add_argument( - "--include-baseline", - action="store_true", - help="Include a curve for the baseline author itself.", - ) - - # ---- plotting options ---- - ap.add_argument( - "--plot", - metavar="FIG_PATH", - default=None, - help="If set, save a Win@p figure to this path (e.g., win_at_p.png or .pdf).", - ) - ap.add_argument( - "--plot-show", - action="store_true", - help="Show the plot interactively instead of saving. If both --plot and --plot-show " - "are given, the figure is saved AND shown.", - ) - ap.add_argument( - "--plot-authors", - nargs="+", - default=None, - help="Explicit list of authors to include on the figure (default: auto-select).", - ) - ap.add_argument( - "--plot-top", - type=int, - default=10, - help="Max number of authors to show if --plot-authors not provided (default: 10).", - ) - ap.add_argument( - "--plot-min-groups", - type=int, - default=3, - help="Only plot authors with at least this many comparable groups (default: 3).", - ) - ap.add_argument("--plot-title", type=str, default=None, help="Optional figure title.") - ap.add_argument( - "--plot-xmax", type=float, default=None, help="Optional x-axis max for p (e.g., 1.5)." - ) - ap.add_argument( - "--plot-legend-inside", - action="store_true", - help="Place legend inside the axes (default: outside).", - ) - args = ap.parse_args() - - # Expand globs - paths: List[str] = [] - for pat in args.inputs: - matched = sorted(glob.glob(pat)) - if not matched and os.path.isfile(pat): - matched = [pat] - if not matched: - sys.stderr.write(f"WARNING: no files matched '{pat}'\n") - paths.extend(matched) - - if not paths: - sys.stderr.write("ERROR: No input files found.\n") - sys.exit(2) - - sol2author = load_author_map(args.author_map) - runs = collect_runs(paths, sol2author) - if not runs: - sys.stderr.write("ERROR: No valid PASSED runs with latency found.\n") - sys.exit(2) - - groups = group_runs_by_workload(runs) - ratios_by_author, totals_by_author = compute_ratios_by_author(groups, args.baseline_author) - - if args.include_baseline: - add_baseline_author_curve( - groups, - baseline_author=args.baseline_author, - ratios_by_author=ratios_by_author, - totals_by_author=totals_by_author, - include_name=args.baseline_author, - ) - - if not ratios_by_author: - sys.stderr.write("ERROR: No comparable author runs found relative to baseline.\n") - sys.exit(2) - - p_grid = build_p_grid(ratios_by_author) - - # Order authors by median r descending (optional, just to make tables nicer) - def median(x: List[float]) -> float: - if not x: - return 0.0 - xs = sorted(x) - m = len(xs) // 2 - return xs[m] if len(xs) % 2 else 0.5 * (xs[m - 1] + xs[m]) - - authors_order = sorted( - ratios_by_author.keys(), key=lambda a: median(ratios_by_author[a]), reverse=True - ) - - write_curves_csv(args.output, p_grid, ratios_by_author, totals_by_author, authors_order) - - sys.stderr.write( - f"Processed {len(paths)} files, {len(groups)} workload groups; " - f"wrote curves for {len(ratios_by_author)} authors to {args.output}\n" - ) - - # ---- make a plot ---- - if args.plot or args.plot_show: - authors_to_plot = select_authors_for_plot( - ratios_by_author, - totals_by_author, - authors_order, - include=args.plot_authors, - top_k=args.plot_top, - min_groups=args.plot_min_groups, - ) - if not authors_to_plot: - sys.stderr.write("WARNING: No authors selected for plotting after filters.\n") - else: - try: - make_win_at_p_plot( - out_path=args.plot, - p_grid=p_grid, - ratios_by_author=ratios_by_author, - totals_by_author=totals_by_author, - authors_to_plot=authors_to_plot, - title=args.plot_title, - xmax=args.plot_xmax, - legend_outside=not args.plot_legend_inside, - ) - if args.plot: - sys.stderr.write(f"Saved plot to {args.plot}\n") - except Exception as e: - sys.stderr.write(f"WARNING: Failed to render plot: {e}\n") - - sys.stderr.write( - f"Processed {len(paths)} files, {len(groups)} workload groups; " - f"wrote curves for {len(ratios_by_author)} authors to {args.output}\n" - ) - - -if __name__ == "__main__": - main() From 99fb4a5e52049bc1a1ec30b51672095826d7c130 Mon Sep 17 00:00:00 2001 From: YiyanZhai Date: Fri, 24 Oct 2025 17:00:34 -0400 Subject: [PATCH 2/4] fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 90db194d..cf9c85fb 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ print(fib.__version__) ## Get Started -This [guide](https://bench.flashinfer.ai/docs/start/quick_start) shows you how to use FlashInfer-Bench python module with the FlashInfer-Trace dataset. +This [guide](https://bench.flashinfer.ai/docs/start/quickstart) shows you how to use FlashInfer-Bench python module with the FlashInfer-Trace dataset. ## FlashInfer Trace Dataset From fdddcc2e4a4671be329aa5b7fbb402f1cb080448 Mon Sep 17 00:00:00 2001 From: YiyanZhai Date: Fri, 24 Oct 2025 17:07:16 -0400 Subject: [PATCH 3/4] linting --- docs/index.mdx | 4 ++-- tests/integration/flashinfer/test_rmsnorm.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/index.mdx b/docs/index.mdx index b169b42d..71e9b62a 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -20,8 +20,8 @@ Useful links: ## Next Steps -- Get Started: ./start/quick_start +- Get Started: ./start/quickstart - Installation: ./start/installation -- Schema (FlashInfer Trace): ./flashinfer_trace/flashinfer_trace +- Schema (FlashInfer Trace): ./flashinfer-trace - Tutorials: ./tutorials/bring_your_own_kernel - API Reference: ./api/reference diff --git a/tests/integration/flashinfer/test_rmsnorm.py b/tests/integration/flashinfer/test_rmsnorm.py index 50d56c4f..0920ad83 100644 --- a/tests/integration/flashinfer/test_rmsnorm.py +++ b/tests/integration/flashinfer/test_rmsnorm.py @@ -1,4 +1,3 @@ - import torch from flashinfer_bench.apply import ApplyConfig, ApplyRuntime, set_apply_runtime From 6ca7bd8194e06c04eaceb015ddb17cd735f031b2 Mon Sep 17 00:00:00 2001 From: YiyanZhai Date: Fri, 24 Oct 2025 17:24:44 -0400 Subject: [PATCH 4/4] upd ci --- tests/bench/test_benchmark.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/bench/test_benchmark.py b/tests/bench/test_benchmark.py index 511b1545..a538ff4b 100644 --- a/tests/bench/test_benchmark.py +++ b/tests/bench/test_benchmark.py @@ -25,6 +25,9 @@ ) +@pytest.mark.skipif( + __import__("torch").cuda.device_count() == 0, reason="CUDA devices not available" +) def test_run_all_empty_traceset(tmp_path: Path): """Test run_all with completely empty trace set.""" trace_set = TraceSet(root=str(tmp_path), definitions={}, solutions={}, workloads={}, traces={}) @@ -38,6 +41,9 @@ def test_run_all_empty_traceset(tmp_path: Path): assert len(result.traces) == 0 +@pytest.mark.skipif( + __import__("torch").cuda.device_count() == 0, reason="CUDA devices not available" +) def test_run_all_no_solutions(tmp_path: Path, caplog): """Test run_all with definitions but no solutions.""" # Create definition @@ -70,6 +76,9 @@ def test_run_all_no_solutions(tmp_path: Path, caplog): assert len(result.traces) == 0 +@pytest.mark.skipif( + __import__("torch").cuda.device_count() == 0, reason="CUDA devices not available" +) def test_run_all_no_workloads(tmp_path: Path): """Test run_all with definitions and solutions but no workloads.""" # Create definition @@ -109,6 +118,9 @@ def test_run_all_no_workloads(tmp_path: Path): assert len(result.traces) == 0 +@pytest.mark.skipif( + __import__("torch").cuda.device_count() == 0, reason="CUDA devices not available" +) def test_dump_traces_false(tmp_path: Path): """Test run_all with dump_traces=False.""" trace_set = TraceSet(root=str(tmp_path), definitions={}, solutions={}, workloads={}, traces={})