From a0d021b03086bd8255d2ca47e42a29c052d3cdaf Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 22 Feb 2022 10:50:02 +0100 Subject: [PATCH 01/12] WIP --- docs/conf.py | 1 + docs/index.rst | 3 +- docs/protocol/core/v3.0.rst | 57 +++++ docs/requirements.txt | 2 +- docs/storage_transformers.rst | 11 + .../sharding/sharding.png | Bin 0 -> 28059 bytes docs/storage_transformers/sharding/v1.0.rst | 196 ++++++++++++++++++ 7 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 docs/storage_transformers.rst create mode 100644 docs/storage_transformers/sharding/sharding.png create mode 100644 docs/storage_transformers/sharding/v1.0.rst diff --git a/docs/conf.py b/docs/conf.py index 33351f98..9f74fa86 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinxcontrib.mermaid' ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/index.rst b/docs/index.rst index 7a985d07..b243ce2c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,8 +10,9 @@ Under construction. :caption: Contents: protocol - stores codecs + stores + storage_transformers Indices and tables diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index bb6858c9..da589aab 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -383,6 +383,17 @@ conceptual model underpinning the Zarr protocol. interface`_ which is a common set of operations that stores may provide. +.. _storage transformer: +.. _storage transformers: + +*Storage transformer* + + To enhance the storage capabilities, storage transformers may + change the storage structure and behaviour of data coming from + an array_ in the underlying store_. Upon retrival the original data is + restored within the transformer. Any number of `predefined storage + transformers`_ can be registered and stacked. + Node names ========== @@ -895,6 +906,8 @@ ignored if not understood:: } +.. _array-metadata: + Array metadata -------------- @@ -1019,6 +1032,17 @@ The following names are optional: specification. When the ``compressor`` name is absent, this means that no compressor is used. +``storage_transformers`` + + Specifies a codec to be used for encoding and decoding chunks. The + value must be an object containing the name ``codec`` whose value + is a URI that identifies a codec and dereferences to a human-readable + representation of the codec specification. The codec + object may also contain a ``configuration`` object which consists of the + parameter names and values as defined by the corresponding codec + specification. When the ``compressor`` name is absent, this means that no + compressor is used. + All other names within the array metadata object are reserved for future versions of this specification. @@ -1499,6 +1523,39 @@ Let "+" be the string concatenation operator. For listable store, ``list_dir(parent(P))`` can be an alternative. +Storage transformers +==================== + +A Zarr storage transformer allows to change the zarr-compatible data before storing it. +The stored transformed data is restored to its original state whenever data is requested +by the Array. + +A storage transformer serves the same `Abstract store interface`_ as the store_. +However, it should not persistently store any information necessary to restore the original data, +but instead propagates this to the next storage transformer or the final store. +From the perspective of an Array or a previous stage transformer both store and storage transformer follow the same +protocol and can be interchanged regarding the protocol. The behaviour can still be different, +e.g. requests may be cached or the form of the underlying data can change. + +Storage Transformers may be stacked to combine different functionalities: + +.. mermaid:: + + graph LR + Array --> t1 + subgraph stack [Storage transformers] + t1[Transformer 1] --> t2[...] --> t3[Transformer N] + end + t3 --> Store + +A fixed set of storage providers is recommended for implementation with this protocol: + + +Predefined storage transformers +------------------------------- + +- :ref:`sharding-storage-transformer-v1` + Protocol extensions =================== diff --git a/docs/requirements.txt b/docs/requirements.txt index 1095b98a..e2adc36d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ sphinx==2.0.1 pydata-sphinx-theme - +sphinxcontrib-mermaid diff --git a/docs/storage_transformers.rst b/docs/storage_transformers.rst new file mode 100644 index 00000000..c86ce4ea --- /dev/null +++ b/docs/storage_transformers.rst @@ -0,0 +1,11 @@ +==================== +Storage Transformers +==================== + +Under construction. + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + storage_transformers/sharding/v1.0 diff --git a/docs/storage_transformers/sharding/sharding.png b/docs/storage_transformers/sharding/sharding.png new file mode 100644 index 0000000000000000000000000000000000000000..85a3c331df18cfe2c0e591384f120fc9d855ee94 GIT binary patch literal 28059 zcmb5VWmH?y)-If&#hv0%io3g8(c(^l;_gm?00n{+3GPtb-JJ&4;#Rb{yThgDyzl+) z-*3#3k&NtQtu>$d%r#}N9j@~E6B;rRG5`QTlarNF0|0<2Z!eN}2yfp!5dBPldjqtXZWdiQ_j?dt#c@Bg1ysbQlU zpCiFuwL`n{GEmVfo&o0jXDY0?bX%5t!))!Qk6q*s%KFO3O{b8xkuL)68q>fefMy?^18T3 zZ$LPhz)M)@K_`9eeaV!3$X`fVpUOgv|v3eqPFnl+cILVnO0kl1IE6= z#0TJDDQk)xdS=IqvA7uCSGqmvTUq{DD_qx%37k9K_4%$2Vk3)&ZvYM6B=H+p#V zlz|RZ0#h&|!7)+b({S+VpHg?vbpih0J2_Q?iozdeUe^K#4jfFF?F-xzySe{pzpm9) zwW+>$>BTj`2y_%in(OvGSZJ%3U-S7XN=s$uwoZA+#^Y*D4eD@s;;Q7B~ zycr7&`viTIhIUW}k?v45?7^+kQi_z=vG7?r8UI(-NbVBAUSd-1Qc_O~M4*-csc$w? z&Fj9W=|x-ni2p;&h_A1IK5lwbZYIwIq9V_*tIr7t{-0%j`z+P2F7*@!zI)~dT(V%D z>tdZRVx3n$?(BO#^aTD_z;{7luQssPF!*ld?N{hn=icVQI{)udS;N7+tm0drwi@-K zo*ap>j0mudG%#(~>`0YG{|A#ZJgjp)YR*1YhkC%Mexeu=_^(ze^JEdy1M zf!1YgN}+VGuu^skQIcs&{ruzV!S+A;?vmUsftxL*giiH9-qT<&Hz^?=%5UWVuQsMq zPs?Dhk_Le5DVPG?&t2-yU;V0zax)V};>BL-$;^7dNE6Mg6YTZocKzVHE%4nAICk3= zO;}s%$wTVtPhjL=KBMbN&Og1=H@fTaFy?sa2!|baD(Bsk9T{|W{2hJ`JTyX3FK6dq zqTVykkIqYfKUF|_*KQ#t^hN3^27DI|zDxc1KE5;i-OgRwi@ubTz}*Shs|T`iZ%s2G zR0){Q)_U!dml6sn>G)6~a+RnSAb)WHn37~!%qGC9B^n%{;a&dE{Fkol+mxG+5uLA} zCIZ?7{lebUJN%k1zs2vvVA=0%^bvDC!`b;BjZZv!DD!=jL*>jJzY%>{wWcH_@a2@F zr!8xovjF1mfB^G0={vRwN=(0$T9FJ!l5d~YE}1KXFB!-B$+=LxtzOCHqW8%CM_+pAyb5fyzVu&}zc zkZ%bCxZr<6UtOi11`ZO|BHxvg0oS>Rf?wibCk(JvIOr!QbrLvx(?7<)Fj4x8P>QsU z?kn8i{=yRfnGj=#*BvXew`?~0JxVwqxo~_v*RWg`fBbemwW~k>C23Z=WPoj#7iEy7 zbxlwPx+?=GhnR;RcLXc&fkKUq8EFAX2p>{Bb|56MhJ{t7H_SEPh;P%x%e4M0-Qho5 zfdIOYZB4e`Bw^SdB4v;~(EEo!DE#Ku!Em8EzlHrN^(A@OnXN(f(x@RGxgC>tab=AUyu z-e7K9O^@e|5n3aU1(X87huAp4Pmp9Wy%Oq_N15+AlvYYl?I^Pd^B!5wFYDd>{Kf_L z-=<|#it|ZxNhx}sdEvz$6ntNJEu0{Y-`W^3$#?bc#|M zcGkJSaq2b#tj)^$cSylq(fTz&fKnFqV9Yqfl;B5Jhg=3Vu@p85#)0Yp&oBfe(-m05ladi_wZiK))Ht>oQpLS5bWmr=CnR|M`iw0 z1g>7-INpd0wYR37~+7NBC<1)U(7gR^tx0k?_svp zWvmQM+}fhTv`EF|5ogX<^es!}XfM{-7(6^c&UfqGSm;9Tgsv{6wbr$frGkB_g*2v9 za10of={8DLU$l1cLrL6hYE6oUj2DvbljWnQRg~Ig7Hxou%2z}1PK78o>!#xr%-^h2 z(?^j&7sAMc0KujAC>lh#k!qI&leVe@sg|;|4f%05YfMT}qAj4tJ48b+CX(0%L%Tbe zBZzGH4!5puBjqJ`R(e;A8cMec3KlcrZqeYA_4i^TKN1R#nXR(l zvVz)NGPsx0UC#LpKQ}B2Nop3@?~p@+ zJ=EhHVlj~?k~Rn>1PmmZFxjYD0H&8byNr>0cFQPdqUNw0VDKsyswib{j`A09P@z=mSP{L5pz$ zBEE)#hP60I8ED+jrUK9$(A!;Mv6b6f9@HW-p+y*Z(#r)4M+kLEPd;M05y( zr4g%P$`3MSxJtMBG$CBPCnwjCFKyl*tx(aAZZ_4#)-?P&gS|ZO8lxaG*Y@*IZnQo; zTlo67Qg6wlkaDr+XWi7Oh2)O*>+_6YSbGx8b)^e|1d&MG9tR2e9G^BB6Jiry%S&B& zw$$E0_8)5xWZ>1rjq01swO_6=mLwzuBz&X5ON#yPGn9oPeo~ECSsK1nC-dK9;_pY_ zGD(BKY!Km1v+G*M&DsRqp#0F=1-y=#bGl$y9Q5ksSgz4vJ2v;vzv$~YisPFIlzA&V z4PuLG_1eK|Alpg!4;UHS~sQJrjm173L9?P-PV~>4tJtBWYm4Kuf z7yaGO1mp+-OZi&YYTA6#!c}F+#s_L}Z6%sJ{e*k^an3BHVm*QCE?{XMhK(M*Mq0K> z^uM*m`V@aRfSGM&Hvtg(z<~MyLunq)J3&f0An+NzS~A>|H4|gkDw_kCk!_LLsRy7r zWS?wk04MKy&512fGQBn^L zO<&i$aa_gF(9_Rdtu8Lek3sMmN*RIYGOv!D?5YDZ`p;`*QIoaQWfa!wD9LO_LpNog zksXV&vbpqjpRA+c z-+j&O-BjqIGc&#xyVCC3BkGL9MT{=N3YE1dn#3v^Jri}#@;BNc#E^Na$Ao<|9MHLn zmaY)9%$EoL%IIg`9Z*P03W~MV4ozQMFKxVMP@=Nfj_9RCWxL;Pfa8rZBrSvy6M-Qg zXE}ozj;+=cfU`W5K+Q-{p>!lDROn!mJGnITx8pvA;WHhQ|COw5#cdi{5g@(Sj+N6h zPz#R`ZWd{eE+&yIt8IQkl+Wpygbhixs(f~1&0T~P5`iv^qd!a%JF^SqHHJWjmC6u0 zwAVkC)+`{1)b)h&-f^6V*6wJ|Jqke~~jD{rqr|Nu5-t+PAE1-xO z@#T7lGbA%k$y_~|TZ>AN8ABRHmfmEY*ik`QIaCY~%~?H=I?~mHyUVW&3FhLCrfR8v zcz5ZsOR{9#yQ&4HL7ok&T+F;!ke~eDkpp41Y7P)dBXFy{%^K{TWr)Qp;t+fp3N-{P zHcEz2CQ~QBrA=NmozA#NvlBkFYh05WPC`;}CrTdlejPH8Uw>BVkYyv^eZASfC8~b< zq`BL!g-ur)I;M5NS)gdhN@-|Y39JBYNpm){fBW$y>2if$@Z5=VdL=S4Y-gS($@lE%gEif9a}F?{3nfC zn_4erS1Go;-)I@+E1&qipAGx(<7bfBpz8OQ1Z^@&zyD+O4NwoW;AsLNTr=-3vTfa? zHTl;4$AmRHFYBdN#-&%D3Dr=BV$E82J_Zw+*=?3!Bc5@2?_P3&u@#a|THEPAL!yyT z%#N69;#?;ywjU+EVdemVExM>O7i98(tDgLQrGJV1Oi=KzYq70|iJB$I_UE^X*8C>u zR`u2kVa>9IF1#x9t#?eeP37+H=AK69tyXW}&b{hQq#EANT*r{MUBB+`acJlauis3e z$b3Hl%SuQEd)lMQI1=}t2g7nB0?@R*dwz!Pcu`#1Gq#V4G#2ly+C(2$0Dj>FzUyy# zZQ5bcJGPw6*IO3wkZGH?N<|pCS}GRBAayJlagU5+wm-}OhvrMH6}PCue; zdD|b4>ij^Z8oe%ya?yjb)(O z6v!7K0aJceOfb*rml#;V#Ui`c1@Pxbe$i2 zzWTPNjeJ|5<*Gg~%lK4+>mO(%Ca6DC>b0==Km*l-0888E-SJ=h8T+DChmq;wE9#d-_1KYVjaY#mWy| zK%3{Y^)o!j{5H`N+5m&vQ9GK-(}1goB)P^z*;#)5{a<`Qb6K}68+>d~odAR$(r6mW zc&qP*$MauXui5wKTX~kEVaSDtz35*=LcBdeS1>eoA5Tv3l06xwEx|MPr=cX^Ea(C7R6yf*a?KMLe8%}4oYo8Z&c49o&Y2Y# z7YsPE+Ni2UxE-g~ep%0{aoU>>XlkhKpcjdGToFOMnx2mad)kH@rCOhuM!a8ooDkMx z>~?B1Lru&gT&TzHI%JIiFNW3|I+5`5RewN4P2gP|d|m}OkSY_4^fu3|i{T(%DLeF0 zo~_mHM{Oa}^TTg!OD5`!Z}c~l`nC`hGX3EO6|^W<)RGrd65uZ; zci*6ucl8HLyyHpLwcQGyAj_ClTWsgkKlcrD3G#2|J&n6dyIjC@DuwRljBJAP{R~)` zL1bOE*09D}98NAiTK6RI=uI1CFZbOL-{>#j&{F3+YZm5tXCNw9t9PYeSMS$n|O~;VqdBf#fuM`-+)8xn@ zz)5jEf7dGfvE>j2yAl!pfcgv*^j{Dki*`GWYW}#e5Y=bBfl14pOJme|;ez_a#Zl*~ z$JqVNR;u0SJbUNUA9f7}2 z$k0x&_zuo)<+K0=m#i}(Wf`6kqu;;wFFR93MJdgIy3FX#bHT=XX5Ru-$5)QZCP|Ro zXABIO3TBCf<@i2={z8UzPRN*MVSc9$Yn;Yk{IIM1366DyT+js{gW8!my2Whs3EJqK zt>9oLXW1v6s#G>G>bzHz4~UOZMDO|+jf}{#T$tlb;NlymK8~yxupB61Fk{a@xAPtF z6WRhtZ8ac)rcbo$-Fj;$>Jx5jKgn=wMlv;@pazRUGp7o=A2ea*Y$wf~yCb4)+$)f$ zyPsQuY@du+!r@~U+TaphN{#2a3(a#xq^=%Wbz#1;=FX=u{;(z@xvqhX3Q{gpU*p|+ zmgJ9tqzAvx7}*ZGy5OdaZOkLH%_w8%UspnNGu&Wm0d#KD@)!|d#Ds*&@!#}Hdo-}v zs>ck>w-}Z%q|G5_1m^FjTDF5Bq)Yb$dy*eFKg4IcD5NF;=pYE2vk~teV9h60(A2Ol zJ};G}dyG6bkC{>(*NdI|TlzgHb0xs#v(`r!Eq~itZsx^1-~9x&OeBcnW#C~oJ=ej9 zJxJa?BADOxXCVv^sDJ{ACU(%JI^>_38>Nh33Tt`pCQcPTpqM_b{ItsVCQ#Xa^1LF{ zM}&i=^D7B&`4m7KN0EIv0B==d zLOr@5c?N0EH*h0?k6Wjvn!F@X>~`?u=k?+ob-A@1KFyT+1zADDm=YjrF@`8WC3$LJ zM)2h;M-JIEViIJw5&qMmm!41dvPcZ`b&Gt37f*tuoR(b3g#IwO*%`yoE9vml*9~dx zY8VSiVVQQ8AKZ&2tRYq!LWo7Qt2H|ez&-h)k54%zamRlvSecsB7$BSpkcA5K>X`FC z_ovH0wH!z>rSvm{z67?GCoaf{8kj0b^xGR6UggGws}I>=Io6f&V41%x?+&UJLH0X@!H=Xq*-*OAcHV z@b}e<*6N$9hQfAf8?dzWu~7Ssn^*{B6+0=#UR^yDD>k~zAr5pY(*G?Y)gVyF{3>f& zVmg0%XxDoq%tR@R{*wG9cnrcKEa%J+FRF=goyZOb1^1;Vwu)5z1(eQiYaNr=E0*GW zm9!%8gl+1qEH41G2I!~(M3HIH4X*obg6>(;e{s-fx+%Y;8VjexBWEE=d(FU?^-kk7 zyN`?Zzu!HSG#O;dQC2$~Yi=N<6WE&{{ZhYF^)Wo@#2t24To?eF@+Q2f;E*F!7v6PD zn1yKOzqjkU1VOMTsE`PA2)_oRa+6zpoWRHd|IjeMnTHjERg6iD7~lQKu6jt_LTlta zp&D#}c-WuI3k6E|Di*8tGf|@;PdNx@DMIl{?Ly_KzLZ*+aS!1PSYm%7c%JJ;h$8?% zq_NO~YIX@ssD!q zIZ_ful{~5b9#Sg6bhjTrG#t%23@G+fy>@=z&m(*@{V<|mt!oAQ-i*u(|9f=R09ggn zV^p}%zJ}oLYUX3LaBS3pm}F`>bC76ZR6wNSEfBOCZ0Ywh)mfhY_m3Wdj*80}V4=gC zXm_dY+CNolbgjbmrRH%f`BtiqN_|Mqrh_osk|nrRRw{oo#A9 zv_t5-R6Zw6k3s6eH-QYK?ovEHtX%gQ@#Fzud`^Ol)}F!;)E>KZ0H{s8dU?XJpnz_f>5F zh@rh(moZ_BMf(gp82Mbsp;BImO0jPpJBvjhH1K|?AB5-;rJl9SFliGa^4mSUgNmAz zno|ar0pL8k+{X}|h6*CU)smYDpK+-i!kZqBC1TbtsAwl1_=Aer{3qVWJnf4-2!xxl zwa*_R-kfB|95iv}sr$cY0j_jGz1vy rQbO%Xr##gGP6p=)X-lj37@}dZQFkVW{ zG{SxY?ls5)VTU-4AB!~#9uQ5 zntKdB7{7M-9MBT&Sme2FbPfz$HGYX#LPCwk__YXYq8hgSKPVAJYJ7S7ZW>lm)y(;I zSo0IzRIq_Lsc6F`EH=G7V&#;mKd<^H#nEsS0Gdm8UwpBrbs8yNj+@g3l8m4`vUM=VZx9=~yI{ zwV%izh137qBK`SmuWHd}|2q-K^Z8ppV3rLpjY{hUz?iX&E0S_1gKU}NP9dG>eXAk{5oo23W zZMzO#3wRo>+kC=7v!ISp2aF!Tsrhfh^p%!=Yt}y%i8 zR)1(n@{6U(bQP)(18XVQ+S`5T*c#eAXs&dYbN(xlU{P(p;QO2cZGs~BG^8?51}YS~ zsW8o2XPYMgqlaoF-+;QPE$3`j2tj!NcjX`;7ZvjoQqRha{}Sqx;zPx;(Dln%^k0*6 z`Ke255!eKLIIkQGz`ov}0%82vdK|I%rAl-@<#f1NGJ4kRV^MDCF!Qma!nlIH+F?e( z5!r-+-M27&rq{n2bLlm{Iu#^1burwbvis(RHl<MV z>JDeHk_SZ+Ef25!EhC)~nK7oqD8<~pKqyfADKAlJF5w~K1V~-{`T`d{Ccy%ss0fM<^9W6X{rXb63 z1URio+0eN2FA7;b{93zI6jM<8A?y9?CV|t+de^euKbr#kUQ+_QoOOOpzc5q_gCpc0WZbD^{OyW|z=8_G8!u`P|VbWgh~i8K^Uh>W)plgQ3${d0yA z+2MBV^q>=cN10C}#p70k>ud6;e}k&Ma-7bIx%ix{NhYgWesT=67AzwWkm()?kswAB z9voVlE^j#Zr$ZQQC$C*8G)vy-!5&@Biv;Cva^>KctKc0eA3S11eW`%XfBF2;kR~;% z_Il-Cqe00@EPLmxIe_Kgc#Sw@dO|7a-s*D`eShYhS3Iz^t;CZC+8L5B%rZQ6N!Go3 z9OmA1wDq$PF~&%DFSY6*^!)0-I?aXR%B1*fN>*2Nprq!tk*|9!i8eLdll~7+cYR~} z0Fhi6inG#6q}0x3ZC(EHGH?3bBF1oQr2F$9+2kn z%cM4OR7JwI#@+7a^xNT^H?Y>_SvRcv9hj`w3Sd#4u%=z)<#E>eG~+kIb4*i_>~qiy9XElg>Oc>PQhA0V!@PHp3lr$ZVA7Ev}#I&?bZhRN~aZ0V1ct|6SSY z4-N@4tu^XnLBwu0Xe2+@f@~{?Rtq}ka9#R4HzAF)GKVhLg(S15F*likEM<9`d5;V} zd}A)n0#ZS3!TRu0rT(B%*2eG2TN#(m#weh;%pR|j>1`5wXU1+>_7A^ocb2Wq#arv; z8dT;xtb_yJx+zW$=z|c((_}A5LD^Xa03j9t*K!9Mp`Xkf=g1v_MIaz>B314MV2&sM zq>DVwOpm#~E_h02RT@rFh%Rc8%QEQgXao?jn{E;nh^Zm&Q#HUuWr#y|YJRcZHDRCP z-?!wa5iEY!Z{dksGhDSoR@(J4-A{x2@5#kgh&2;*k1+YOMbO*^O z?oAIR9Oaf-Hb^R`N&G@_dX77PFau=fY>`ax^vqrsq=JT~y8pw~eB|wkWV=e2|8bs4 z>d0w`wWsy$Z$l7}MFW*E(q&rc>!>AX9NlFh7&p&iDrb4C8y^tQ-ANTTwi*2?yV!j@ zUoA6;`@f_42Yse~+(|P~uDB&0KtsQB>6WUx0T11%l!PQ5Uc-zE%Ojns&s(&q=!dXp z|IY9W*~s*>K&x4vWB?Er(A!n?KXKohC##~lPU5&yaGH$tQqr71klA$C$R>-Ge#`3$+|0{cZ_HixI|sw^_65SFw90=y?eGQm|Lv)>l^2sf z@auOLa~c%;oJd=xQFgUKv;tN-t5k3)L26E!G*FIG97h5S%5)?W=fvy8bABIdp%QH{ z;ls_KY)oO2UL9F`@Y;r*)QUezYcy*CHW-6~$2`*?c<-h*UmU*vqNn7=W1sEHeb#N= zbv^XgurA=`1~y1jsx^RkwsbZd4QT#05#HYp5310`D>w* z%$7xs0>%!%rQBujY*ray#419#{Aom0}}8$%W&nGHo#zTh^& zL=7nq`;(ao4e1RXEK;p)Nz?EqXd+YI%ThbIn{%=k)W>Ec#0=FT#1~*KR!q1b@ZnSg z`Tp2omUihQ>1+UGp?hM;$K(q>iz6@GM!!g~0y18kIv$UZW-7l*TZ){`+p| zFW;Hl6*4W-cB$*?uZh%^teMHvH_nLuV!z44#Z#5y1O5V=00f^6QJ>p`#tvyq z+|}gQEIkNn*o4h$=P^mELh|qC13^P?2TD!#KezrG5VDkdk}uhep6*#V3GR8etSKnU(?C5l*!3^tx;!s9Rq8=PrsZ^es4Y}W4d+@1c4?5RVVm$+cJK(j|`DU7Gb z>eDk&)0MK9L}frls>ayZIf7y#sXomlQ2be6&8gnt?2V?w%Kd{#NHm zy`oH!FD1N5w=JrQeCICL?7pdu5q<*SUvXNyyOiu@*Lv@8-Oh`~wiyWxskU|D@9^-+ z17rG#bp*d1Htzmw1b!D%)%kZRRXYo#8~hwc)7MS@wu zVC9&10<&ZgCBp9ZZv%2sx=f4FIR;95h^LKtpp=Er)MK-=qkU~$gzj(3& zj{g7&oY^*EN(z&v-jWYpy0o30Iq@|26v7AZglkOq_n;Q-0XduevP6~;@C@br=lzF) zkn3~-at}v9yfamWpJ#NY(3#h!oN9dNUW;qtqwgA!7*%^>;A(XboowPpeiLrrB zi8wG)V{XI%q4@ie5sSwAoQ^=Y%@2Lj!<$O8z7Rb`dQt{IT1nBETC&Y;C;@bGUS7*X zOo?rl+e|Ra_k>AK$>^Ee6;b)-_OnA~a#!=O{h#m|4L5&U5WRwXFS!pERy~b>n`;If zi$lF8Y{ys5n|}ak>~&@wG;aPe$(1U&Cs#nCLTCZT7%FL_pdNlcC(clPJQ9(c9q}^^ zTBk#}+O|bSiWn#rA=EHdrd1>X>>4g>6kg zh>1G1-NS@)dEpSVIP!dKgi6=DSWDi`M4D_l6y1mB&KzSg1H_I9Jj z5g#jtL591(fo!u5Jupy%5oxDJf?bvi$so{-J4eyHsq{{{a{+e_EK6>eBEjeb{uHYTf625qCPgdg_w|mx@2&EEa z+={jDfatHQ#isu`Z1pd7<+OB5PWpOuwR=ahFK!^WKw;=dRG8Zg+Sq8Uge*rSx$>cw zoa_6fm8}JSY<3Rl^@M;V0p~c#YS}n%irbC_eOX(_7=< z+$j^+;gswKz}A<|CSy@{LSKr&g&m{yUPebGq{;k!9YKeM4y)vZBR-9){qUtZA5In* z?9md**lj_5ay31~9!Pt>UN}&LP1*L(`Mx`+r1RlrLu1)C-wn_sD9g7BoK!`6Jvvo&!~TPa1zH%Ucn3U`Ag z{I3$WX<<}VMf;feCmVf+9)NQ=j^9aOBj$CyvhImkwT)w1R=W&qx6W_%(}MsG zlkp*h62r4@6RE_iC~_U=v(l&&Z&p%h@SMIjiv~t%V633pPeTxOEx!R2^(U6FjrZcc z`>zrfvl?sP8#DPpMQVN>1wxr44q+iCYSbOr|UQu5FjGSdzjm%e=0m+I&&KtXu-Ktw`8Zd2!9jaH65{V8X*taC*h z=hyhc?;>2Lnl2_E{E9Vy*noPBd~Wp06P2y>TE(z<8NLJA>@&?2KpWGnfon@7%6h8; zS*UIYHCfWy#Fi|VbRKe-Ag7QDfCcL|di!rj#%Lj`o&vzH70_#i-<6K?0RzUA4<|d5 zkrAqr(6EoJK^PYnvS|bM(64+H*}{Ztil>QY``!tjS(9`**>dq!ZDKRy%+GK|TG{(6 zvGWx%+bsoJ-f9;dK#Qdl^{Iq#nd)soN+Ve}t+-!ExL1 zCEq>WBjfq87NCyyV;6h{?z3Ht0?>kSmtfz!J{L*w4V-HkDM5AKR61YTsKD{Q;zjwW zzxg4_LQWFs7BA~7UqV?P?U%8(rf&|mCJ zYpy0BAP$~irNBhd?X+Mc-duupA>GR9H&($|ANv?Bn1P;=sRd+_Cu=Bd0pwUlo!wt8 zj_X$E?(;qFe^IrsAR8|Li*0YjAR7mqc616r8rSBAf9)Oy^OO*${}{34=lv1q)vXUZ zm^~K0o|aDDpH5)TI%rhU?Va@QrZcL~sIe%A_oO!pX4+>2e_!VndRzG>kyq;M3xn75 zDBmoXpjVu4(<78seK!zmj_3|9bWGIFr;*=643?q1FHA8_}8GnP&fr`&R4J37e zJ&Elt3+Y*1(9M~*9*tvCzJENn=ISu*z%icjoYd<4({*L@WR()-!bKKoo(nzRBDTGl za#wS{sW-&MR`b@15rlra?Q@E`{#4E6nvJYdOK zbFV@SOiS}hH1EL09w?ciLm~nFY)mv8nk}bGxz>;sEIo}gNO)JQU9S#u7M@q)_?p5; zq793Gib0WT?=&V!U#a9o#nj%nMYhD)&80bGYn)%($1%&|9wqEE z2ir~f&;&zpT_7vtOEJa5jQa6e4HUN@EVn@kq`4GjzpC@R!u-OCr>bn@;z1^$)y+eV z_0Zr+I7^(E3Kt~TUSB(N@gX!Vp@2X3gaS6U@OA^jY>C@-S{}S56b%m@OPhMzdCcDm z>0||lWtFMi8#AWkq7>-h}LCRexhq=Gsrxh>uRgA6rTn1KFh#x&JS^2nsp1g|>hN@sU zWG&3l3E$1s_u*!IwN4u;bkMP;CJFahZKbkL>3N~U2M|i6=^#P`EZr53Wgo=qr*vfm zEy@RVpgawTI$Z4}Wku0l*L6tfvhi+X3ic$_5!|a+B*c48DCE-Oy_3VXqNY>wGnN+p zB#$=2((NAzf~{^=NZ|fl47Xq6YgGkM_rJ7A*lbhYFRravu6EFc=1^hqZHq8fceg=3MX+JRPcFr!*> z#HZM6%qgkxR?>X@%( z2oK+BbYo}9ivfpwfY!rqzbXlnmRh_I|HO7DybgdGTsk#y1UtPx&=R&?s31(Unv>Z4 zoaWX?dlWcTXNAFPy0TxU0N8^VYhOYcun)N1ksp4w@-r9#j^{g z-DpZ?Ys~~}vK&5z2{U;=J*j@TGV`N~fdR$De8E#ZyMY)l8m`iengB|-SuA?6pX1Qy z2eq=pteL+*@3KulwWPrr4_i0F87tNX$=v?cG>vs$HYjB&Moi|~gSqi>Jz|DWN$_fB z&?k6{*+f7Du{yUPMUf$c6Cx5$II@#K4ngol!4rGOifCa>C(KcLt( zk!oBE`gO{!5?VVVgxCTq3sGH^5oR0`&L(V&FM^tiLJScWwb}ey=U$ zQFzO^iggL-?yC`h_Fyg}{Btcwd|rKoLjnDM?n`GB&<1~hG|L9H@8~}x1@$3n&okfX53s=C#)J@o?EWYGA=gsz2Ggi zpTV1`&f_EKyMi_X)3^a24Z|5yd5KEY31fx-^=@_smJ=#Gf4Gk70z&!Z189k9V`{bJ zv)wau2R6ZrycmY1a!>fA?_N;g{nILH-#}+@C?t$ZnO$*SR_zzTbiRT40jGtH+^qi7*Gp5_ zynUTZ^l=zA-SR?SK1Cl`AJLY4VAO;%!3`gv9@dV5ITUS zSRI0~*Ml5+L0b7)nDrS?Lu)Mx7Ylxq#$%A~n31wauz^ZZ#ABP%T)5=g+nI2BQ=K_L zl!$x;G#jyg^{)Hh#$&+Ffk~K9LDiF~896@Lsuw?-)Pxi&TY29VLGbM;9-N_1=4#_I z+{pNa@y~a8Uy;Mp@m<5QUxTzRD=SI!PAhH~BXYM3Qj7q3@xm<7%s2ew?L;ZmlRs{5 z!EwUDw#`1&)5`V^)TcjpzC4qP@^d<|H>#>u*vsIY0Z)trflQBX$TCO?{x(6NY``RUQL#TP60q??(#L^M-_ghvIK$ ziX=An%x_35mSX2~Zm7=5A6URr30%&Z1tGB+V9jHO5~c72J3RcK!oD&ts_0#JhM{{v z2}OhFyX%VQ4{8N)V*Gdr(4h0O{^Vx&`h=&;Q(W?w9-9^J#Xx zYrXG&*IIk6=Xq8m&inl{+hzvKgn&7Gj*fo3}y zS}7A#If=#?PjI{C%<}+hy6ORuGJD3#age01#G&pzCegBxs=2&iYyxJ-U$&ov4LJoY zw^Wew_NDvP7H{aPUh`g!LpIn;e@CeurpdhT%^38U$(|F(bm8~~6S_x87+;IxHQYB< zIy|HIIz8;VX|+th_s=fYld%;>n1*k6(h-?}KE+lj`&i3(Th7|hT!WzUZFBsn1E=5h z&u!(cCxoGZ|}@GdfDmTzl{sl8!^e1;6USBZ{+^zD2Y!c_v)nc^3u1Ms*Q(OGp1=h z!a|W7Dl~!?#U%q<`_)l)Z!g+C-a*cDD5$r1tgvD*guIOe=LLgn$HBv5d6fhA0O)-H z#QAK0$pPXs6mpjvoN&APZZ;{Yvy$zm>EnyhY@s>D^wN^;~u|I}wiE0o@qsr+|mPMiBkzx-Wy z_*+nqT3`5qZ3yK2VYS0}PMTSplKO_-_lR%rLOeq~3#bvk0tbkvT;&vyE|YG^ z3zZ$tcK}fpqdUz-kq^}iQo4}viqXPWCfF19`b{kNrGLjx9=cUAotVpp_E^q8ZmzDX zRnBErkobDEsUotlK@8cCB&j1NP1qSuS}1CWiBWhtnSJwNr~~GOZ3mkUYF1=y58~Zl zB_UN5`B+;eov^Bzt*uJ`Y`QU(bD05DZI#ubSjIM}v`6k)P|4_~B<s-M6y(pht$)Q+{-hg-hf}MKd(~Y;HoWi4~?+p&6YE zPu+-(Tu;;`_V{H?N(RJg7s@SjjU69!5ah|wcDB-ii z_(%Ad+tSyQhs~5eT5!{+NIRbWXBqC5mB$r-7>2e?9d-qt@+i_zA<8Xf}NbV-`=3FZ8xE1zkerkeg!@7SSxnXvN=f;gTb zX+QW!^0UVHM{)=rv2JVqdL|Vxe#I5e@lS|m>KUp1_VpLWmZ=sYq0gd!6*cVV=}y;B z?z^A_B0u<7;GMcT+&)48*%7jPASb-pDU1Doaw8v{7(x#P616^)-Vp>YUjEO;LO;O% z2loY-6O!H>*L458HdS~R5F;rCigno4QT|`niSg)Zpk%-*+2bC1H`>4S!ZAub=D^D( zFt3;A@}l-Xu@L}FyF&|o#4a?xlSFz)_A1H4X_5P1xv2ZVZSA-2S=~ei7p-^7aliNO zKbj&fX@fB^*9`i93Q+~!`;gVsX|12j*s_&tb;I+$VKu()@UIR-@vwb3HzT?LTCa{R zhvPX(M?{Pgz0VsmI`?vvyM}bmi~O}Y`8on5->$(!*wrce+tl`<-z!{<5<3_BytVqI zn8d4($xNc^9n&7aFttt{S1xqBzNrNTsT#e>7q))39^RLRmyOHQX_jIx&j6@%#|MYw zN_&P?ii4l(F=tD9vtAst^ox5wu!YjBM!%;gFde+si2&n-Z`S+HUgv<=2g=>QOl(o_ z9S`C6yjyKrs2>D=KHB3Bw~ zric2`K>c)B?bCv)jMYdPE~=q>&d@!691zM8iKx4GG3!uUZ2F~#B(v5HLBaP?#isC6 z+9IkjNbn^08bcuSptIL6sOiFP+|>I_?BDQ+gzNgIR->Oa8q zeGD@I7-((~f!|iLtwQlQf6{tO^Idp3V-dtp5`}VOZ~6%s-r>M%1j3`UFqsp+ zw5DTl@K%Tt)mi;x9R9{GGs-g%b^586s(Qvc9Q!Lt;Yp&^4PCATAgukkE7Lk!&;TTA zEHPbnS~#SkiC({_mK*T%bP{WdaFz$a$^;gTU ziR<22@>oZmwxCYQ+KgR6J>v$GHM;`#AYRdjF#4wbH~)S87y66v3|3ttULJa=#2qL` zEg2UVoxEKcPGktdy8?7|XrW{VXS{!TG|9e#q~6XBx+??Sx4lUkUjQOG^2k4itK35& zW>PYO9i64uMUH5O`O)IOwmU#S9_Y0=<>WZ$6WYqi8f9rkLJ)F2j;fK9{xdT5!Ilt# z?P7D=Mf5}wpw3E?;Q>S_YrAQuL4RX#7(&=T`X+e65IZPzDl}`$b0C()YX)`tA=*b8 z@#iLAp;HHUpnIWC;j|Y3H{BmHdl!n;K7&{WzUQiyekc9AyO;_(3;dwarsDKfPCjGI zZD{iow{#$0c5FD7O`b)3H?&5-K$f zV>;Z_2QXicqI4l(_0=hJ{xqSsJ?!!wq#6wdF%fKr7B&bW`B1daZtyh}cvt!kY*!(u zn%S;EF)Qg3U2#9v*p!ATB~Z^L)4~~SVOmpsOxVYMj*bq*i~Gfy&k7$cDh^|=2iV^B z@^lD@7M&tpSVFNtt2}aCit0~VQhh&g?_Ig~kCLohrkpU3j>$+l=RwGHErU&c2%?D* zW?(1J)GqQ4V!ndB>by4_;77ui3tJSEPiggyw#7mi&q^@yV+!<0Ac^pk0yoOuVpLe zxVz?Zt@TBpGKZF>8zV&A;#Mb#GYgrs-=qlhua{w|tTn$0X`H>}t!8!~ zT~JSu?E4D3n|OvpbirYgMzN0#tC$<X>pz>_U4fv5-JP{BRd25 zKxPpH#@9t645wR7cfnzsqGqF0bYyk{)ni3z})r~Q9H^aF#gdZg$3#D@qa}{_AYG;VI_Q9ThVv&68&T2fPf=aGb5k8hAo15 zofK&<+&79K^Y+>Y_t?usNW(8j`}|cRQEs#N zh^4pZi&o}i?oCfS{kg!h_v881iN{l75sx4i9Qb`=n^`-f;M4{UnTA4h*IDymd=7_P z9}`;f3VA!fPUbPyeq$mZK#2;$gghI~I`-)H=>YsJbn~c{lkN;f;_*T(V&nc_a_=!S z@&~z00S}KctMm&Y$K8|1vsAm55jgu~jV6F_mb}`7VL{${*XXv3P;>fyN~Y(%`!iP} zjaxr89iBGIzmjpF-NKqxr{K-IU2N8QfBN~SuzYjwq>XQaP6*$VGey&R)@=nI5AMa+ zbc90lBhNhm-|a}Zddaj}Lpw*FRSt!2oxq+`nWhzy2xt>qB)h3UApb{pB`?CHE2~IXe`qrb6%aAz#_f z7+%(2*`^-!?KjHb5@oUBkK>9k5wJ}MqCd*C&;s3C`%mHVx;FaFHyg`(X-*QM=~u{} zl#nUC$DJG9^a%BekhNx939&W4+*f^96hhF(zw68ysT5In9y7}lhs=|$D;{kCfwa~w4E`3<$F4UxJ z0Y{!Kqj+tHB%a6@zO!UQ+Z!)tEtVe{j3eq~0(G9i$8D>11~7X|RmY!r$A}t)W%A9s zqXY#HmkDduZe@NKdr5}&ejP>Ub+M|0EG4`g*>Cb&?=mE;Z?TeoFYZp7pg#FUNqD6aqt1=ZWLwkoh;y7ul0m`B#y3KqCm2Z-)8qfD!bsm`N}_Hm+3( zD#kpj9=|)_x1Y4P-&i6)@<=&4##Dlg2V-RbR_KCL7NR$6r1cPf{zl8vvrr#|T5#ZsRn5rttgsV!D-Dr#11tAXdd zBL*U#B!Vy`1uH*JBJ}#b9Ct|^Gy~r7!3(dhsrlgvk|h|AoJ0yWvX`atL@$qSGx*WI z3P~WioHPpRHG_N38yQa#p?QCQ!hI{Ec*vFM=c_5*pxfQ@xZ{tsv}oB_s0#hWlX+1u zY2K~F4b}TPnw;XgJFz6bUs3M)DEB6qfnj|JCpgfG%ec9v28Y(U^{s9AmbS*@QDrfZ zl#%yoz?I1I8y@sWow3`r3Z~9YW*MLN3Zj{O<)s_?+sQ5l+vrQEk!M8<1PX0WIAm@a zY^JTyyp0+ejEvi#E6-xndEiqg?%8aBEPb+svjSB6Y}Za_!x=S;NT#~)*z3%VoS*)I zYZiqz_2yXhhx<~CfqtZ=ZG5TrE5f1q#N{#|6dSwDa^Pe$N&kA`xo}cRepA;3k#e7T z@iKYqupgO;=r6^WSx%&uj3B9KlUaSZ#OtB@26bTA^<_cpildEdm)?oG@L1hJyYSSy zQ7bh9YMC|50Yjmyq5Q*pIr(H_)M${qB|)k@Y{VtP>H+9$vyc0acTIB8lfSPPA0Lgn zQFOs>apFvY$*ieUogbT3$?^gFC-n4`Xel0rpJQ4Xg1<@^AB`WTp=ZbCV<=scuU-ql z$-*q0eYbcuzZV{^WZ5DyGvGfXrddA#D74bYQN>j(nxecvr@_N%ccHVsy^5R|Tm((D zM9sW5mv4bDU>TUoV}DddZLB~CCn6UV?P{Zc4q+UN&wR1n=pq^?uU=AMA1Fp%Nh7}h zdzo|ad8TQp>(;ncr;A2_sqQQ`qNe)FsPh(Fe?)=!C`>`z3vc(5@bM6QHSdp5p-|)3 zjuaw9D;xfn5%H2RgZt400UG~3rH!au7ae5grg^vJLDPKkQ)Q+}wH{UE(Q)q?F%6h} zuB3j@eI(pb^7yxZ&u!~UvD%#knw*Ze??NsHisl?pmB@;Z#K-jo zn)NO#B3Rje>Gujkt#1dP*}yB?u;hs!>5 za+7F3xb(T%iG&2ZtXQ1_Kakjo0Rb$%TPRxkf}yQJbCbaBkMQL%mL9nc$E8W>Sz_oY z#a(hG{NO|8O47Vl9&QKX%Pqdb)+?C%P4l)tVfWfOZ-`F!qPM#0nl%!Rd_#$7uo^C` zs!)x{!l3YLf0})@)8%%b?*P%%CXXertZ{{!s{<{aLA6!@A9qe_Ti~uqz1f1r_sUn~ zrL_nmt?|!>TTkfl+*5CS!Q7HsBd8Ws04>ZlRo+vjG9{W*`E2aE`Nh*enmGz(`m%!J z9-~hRX2ff`GI!WrY;XTfPLrzwG9AA1%-pp8w!B2fu_)3Z?dJ6pgm+7~gOon6ZKqj* z1L;#nDcwwc{(P4Az{|CF|LDNtyAytYBq2SpJZ~04`=gi!Dn_bxF09@`6+xa%T*EG5 z9Wh?`t}E}yQtM_`$L)Uh=%K%@s#z|}X*H!^RzMA4_swqQoykYPgR-Mv1>sL=Vh@xA`mjGGhG;SP zepMz@_fUu2sk7k3cr$b7+1ifRfp+K5E_nM*pYkrjWE@ez>m|8ef z#o<7$b_DbE5b8PEW|E`7Il@3@%JVM=2L$ zymdyEHCTzwanobD>N_=CMWm*1+)UfAtRB1zX0Mhpg*get&jYXf@|$83+5U8=9G+@H zPs4=^B{m&VU3f}rs`=>>mYyzLR?McR(-=&AyW})Y6RYVFuU*u))q}3@FK(#sPjVi; zfz>02xWp=MA@)-PJdZz`rH;MLFy@#aDlskPiGI+SJN(uCz10Isno8Tyf%w%Zi}R6E z`!F|PuAWlPI^aVq)%sKJ%apun*kx2rcp*4&LaP&^j16GMevn)@-6MljGW#Vw5x0_` zO2GNS*{q9q+xY3%h|FLBwqn7C20c1|tF3l2=Zg=@0X{qQ=D*QjJ*B!|8~nNd_$H|; z50kV)@#zH%jo)w}9aKIPVUzDXlW|o>rkAB(*(m)Pd+N)*?G9qH{@T`_U^9xz4alV! z9n6mFN=p`0zG+1#j+sm|764Q%92{7*D=^pb;JTnVEaIowqq5^#wAn=|If}?o;w;^QN0z?9xuxHY zrIK8@8n9#`G)M0ZzRhbJ2*8>asX($_E<_JOPTRI!20ui=ftHDfv!AtV(slI?8B-}G zywr$I>xuz{$SnwV&tO?kyIti>SbYg|Fl+9N#w1WPtH+Qv^LUG^P~XM7{1IK!T@;wj z@oMHzwA0lpxC209G6mgRjS;Eix=Wlp_$?AcI#e4}1G&Hu@Fp9-;PLvYHvdZ9W#D0N z9f8*6LSo2tCP4Kb&Kfhow&}xtR6WBfR(C3Xa%y13j7+!sO%j*M8tvEfevFB5#=q3H zBsg02QR*J-la~!V`d!ackkMiGs4_^l#4@zwtDf7?r$Xa-njO#ZKU1Q)E)DV8uYTNa zC~?9f7XNHLZgnELf$n!d3^P>*TbEc}h+LI5_Vf1swYgcR>p-x(&IlYwiVuX-}nkB<9)%M@K>YA^N}$E;4WB_h1z_ zfh+V-|GXXne-kv%Jtfr``o(y(ih&}C3I7VHXVM9N|WoOH$tSiG9r zTr=)QFKJM^9p5!UOcDMjwq?7%MJb$AxfUM1He96+&E{~X&Mp6|lA(9&l{ZijYo4i# z+;|~yLfGxs85{dAHNa~9Clu8dtG`=p8Q5|PN<{fR^;=d?yPk$O#lX4E1AtcLwN)Zs z?hKcaEd?UAp_Lg-g{zy0U6fL0g58l)2yU-jUXJE*Ad^0iElL~DxT%>_@C8S183Ed) z0D1elofueFF?Oq<&nx&RyzrM*&qgDLg-+!IT9-7fs;|)uOCYK%qQsOI37?xx7fe96 z;)}{lb=`}OxbpY}`azGoCeJg}tfyR@qG+pu1_t%2rAL18y994;c`-D^Csi53 z9Quv*uh~WLVNw&b?Tx~B3L5;X247OYbkjRVfH^0s;#(hx+RA}`+)GmW@_@vRNX1J- zmTR>B(3k>&UGUjkS(SeB2^E05-V-6t3V61B?DakCv9sxghJlH3X5KRGlcbf#MPCdB zRIf=Uj|1ht-Krf#63FyXkzA|-P!u*8Bz4-qDtxyg0Izdn&yPF2d!#gOL$8C(2%?Qx z4(x!~@K6xn4v2jQCoQBmXmV0*)E3$Cr{Is-5TH?s>WsMc>#uR{3-&6t#lprNefKo> ziT&rxM~C%XS*}m%**`6CC)z)?b>a(}DF@}g#6?d?0aGHJ%R^`7MDrtkDeL^1q0r%3 zDGignT&_-M_5_r#Cz@t8 ze_E{kBn(b9a8A+%*o%InkFo6yHlE5O`9RosX|O@QdM#ig2#1%rWXgW^g<^s`v&ZW3 z%M~rU6gQYb+ft1w=ZG$q+!q$g-&0<7t(SJXpwI8{JA&ya2odiKQA)`q*+8ci7NyX@ zoy)zR{z5|2rjc1Gj5(3D!|H)NSuD$(OqBrI{-8;v6Y06_edtZVOC{yBE+xGeie-0? zC>tb;B^i6Qf>Kf(vEK0{d)9hR|8iNdFSX0VoMVBLkrGP|>sj)sGF51qkG;!|3+Xr^ zB8-DA+YItu7w?r>vXA$`y^{uMvdY$r9WIh8GS}1I)I7MwC4p&}Oku1?Twb7({N5syv!1;~kP7mv{%{>h} zJ?@{wC2lU;fcdyv19fhDnrmFpl0c7Y7jAKFu)56oRLGj;@Lqadx^jX#t&PjzPu+MYNKdvusGi9*k z98tRldqjtWJLgE_?88s7nG(#wH#$Hy@<&nFV-_h05585~sx8&R0{ z!Aw=&aUZ_+-v8e4XPkO;k61H2nq6m_tuiF-CKhVPm@L|8ys!s@(O23B0~e2#b@JGw zKj(y2#cA*yHDY=Ak0iSw#qG)pA5!Ccsb23d-43l;@;Dk8k?GGpk=p;(YjiS!jVOEg zX*d!0xc?4vepBg#WTVuoI||syp>^Uljdk zb~Hz#x1=XSc7v&-H?NWkx$i)UyI3EMc66&u>`ugc=3>)S{$9-wL!>REV} zOGVDE?bJ8F)Y(aU%g9rIeRl{H^bHR~BQG*EyR$db@cLNnTHR8+)zemuV@ePVqrZ>t zWR(oTi1W^;Zb{UcP{NpUZVQl42AJT^u{Oz3q}v%im7&veZv$RoWL+LP@C-=;}_*>C46f1=f2iyoW)M9U!{g+ zqCcznyLA?`C_|l2>fsmHrcsi#cG-73hy26~GpvJ}4h%lSAGiL9a%ckjs>Sq^cI0&Q z&jIP5^rcx`3(>QM)`ZBmjSKp_?tH@-!h%-xE;>d~);drow=@=zY(QB^WKxvPIi;+x z?!PS?-I99AOXCG&yca|bj<8lRlBm?Av4{Kf8h@@duvm4?hmHVX0+_oBfoF*V)IK^-ExM#2X93* zZl?Q$XYq(tzDm&>~enfn;*H05IPUcfx zJVO(BaI&@GM6{wg%FtF4dmJcYJr!1N(D&3tqL2mHZ(~+o zIcAE^$x%&W?|q*xj9HOKenRb7>OKsIK5OLJk!BktICRSTcKL=aMl<@=zMw_psJ0in z29PU%4>7914lCSFTbFM6VHROI*)mnSI%T(i9=h;m?GqDG(udB)vAC*mu6ufYXJLyI z*FV8NF?h4qA@Q`|`0lOol;=k2;N6DY)32J~G&0iZw3I33J0BhKsxrG!3XM=2XQL_CORL@xhM<|F6eD z`j39BPh5G7CTmyCP1m$fFQ7GjVvH$=$E>faN1m~4ybi%CLh8lo%72i3BERgZyEE*$ zeWNizMfZg;L)~CfqWVsQuIR6t_Bk{osr9&A+pT@pr-@UQE-wifD4|?bfnKH@B622m zI^=~XmW{`NFd>47^rt-nT2bX^E8<@5UgQwo;?IUxwe?lQ1WQ~+bJ+Txh8_>j2l+Gf z*MeC0VSLd~5nqoC3P>4a^N2%g^GozE z`71xR&mJ+A(L>7=1PZEut7^FKFja2j9BU=6q~qe5^foU^z1eAY_t;}071qClqi}{L zl2V3!3b?@rB$a11jSX?M{kh$5w>WpFQw)ax5kvZYuTbZQYxSZ*&kKjd< zwlTm#=Fwe#(JTsLg9@z=2P(t?xo~~u-45~1DT|JXD3UL@x$7tD7ZhVUj|itm>IdeJ zNJ>Gpke7a|vbtnr#9zAHNiM&@O43AhsZoIu-+ZXI^3@a$_c~x)k1JTkL>$r zscgoa7I8>RVAL|wYJAyv5Y;~aKo}8?Y18$kKezMjp;|4dolQUHd|SQw`X>z!^PXo? zH(FT@4>vQf(r223qB|?Ot2273P$-%~iLqcrgXWib0GkJo_>f5H60?(a<@%F;VWz~# zMzvQ3OtFujS4C*L`Zj7uDPDKajc_ z1}t;_J1&-y(UfE%_<8eR0h*e^cAk2f3~S$IQ}o`_zY9iN{O<$uGw(ABa1&0r_c*tB zcRcR>p;h7g3z`80*gryMDWPN>3BTxQ^T_je_tXA1pF>V5&J{ZZyuz2x5tV;Gb z^7B2{(uDNj$+b!m{S|MARxL_ntlWe>zmV(@zK{ z4q%VL>;OC{l5-&G!gSMp|DRTO3s~9k`fGxcX8uocQJ6-h#?@=mJ8!=}o^~FQw>aa8 z%*9MEG&i(QS;e zIoGP9s@m2HJ{hTh65Vdth)(2&DTi0Cdss!LEOu5r)S_DK4)C#RKc-aR%BXAVo^m5P zE_@-8*yq7|C3P?d08%EsBJxTM^)25Q;wDf>phk;BF6o{x8^35g9`g*yQ}y+nj_b7Q zg^LM3^vq1B3SJ!JD^!#07Skzqu^6TRHmkV0Ge`*Hr0o z;rfroscjM72jj_J*HbVnTR(+Jrw*l)#Y}!aiN+7u<(F2pS~m*#f7RM8NeiGP#h>{W z=l_!jBMt~q&y%b0^8f7Q6k!kiv3BP_nP5JT1|^i2xac|m!;}3z`(IfCfCm15XRq<{ Ye+89=8oUq&c3T6<%P311Jx2umA2Ze4Z2$lO literal 0 HcmV?d00001 diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst new file mode 100644 index 00000000..2aa7d487 --- /dev/null +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -0,0 +1,196 @@ +.. _sharding-storage-transformer-v1: + +========================================== +Sharding storage transformer (version 1.0) +========================================== +----------------------------- + Editor's draft 18 02 2022 +----------------------------- + +Specification URI: + @@TODO + http://purl.org/zarr/spec/storage_transformers/sharding/1.0 +Issue tracking: + `GitHub issues `_ +Suggest an edit for this spec: + `GitHub editor `_ + +Copyright 2022 `Zarr core development +team `_ (@@TODO +list institutions?). This work is licensed under a `Creative Commons +Attribution 3.0 Unported +License `_. + +---- + + +Abstract +======== + +This specification defines an implementation of the Zarr abstract +storage transformer API introducing sharding. + + +Motivation +========== + +Sharding decouples the concept of chunks from storage keys, which become shards. +This is helpful when the requirements for those don't align: + +- Compressible units of chunks often need to be read and written in smaller + chunks, whereas +- storage often is optimized for larger data per entry and fewer entries, e.g. + as restricted by the file block size and maximum inode number for typical + file systems. + +This does not necessarily fit the access patterns of the data, so chunks might +need to be smaller than one storage key. In those cases sharding decouples those +entities. One shard corresponds to one storage key, but can contain multiple chunks: + +.. image:: sharding.png + + +Document conventions +==================== + +Conformance requirements are expressed with a combination of +descriptive assertions and [RFC2119]_ terminology. The key words +"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in the normative +parts of this document are to be interpreted as described in +[RFC2119]_. However, for readability, these words do not appear in all +uppercase letters in this specification. + +All of the text of this specification is normative except sections +explicitly marked as non-normative, examples, and notes. Examples in +this specification are introduced with the words "for example". + + +Configuration +============= + +:ref:`array-metadata`. + +.. code-block:: + + { + storage_transformers: [ + { + "storage_transformer": "https://purl.org/zarr/spec/storage_transformers/sharding/1.0", + "configuration": { + "format": "indexed", + "chunks_per_shard": [ + 2, + 2 + ] + } + ] + } + + +Sharding Mechanism +========================= + +@@TODO + + +Binary shard format +=================== + +The only binary format is the ``indexed`` format, as specified by the ``format`` +configuration key. Other binary formats might be added in future versions. + +In the indexed binary format chunks are written successively in a shard, where +unused space between them is allowed, followed by an index referencing them. +The index holds an `offset, length` pair of little-endian uint64 per chunk, +the chunks-order in the index is row-major (C) order, e.g. for (2, 2) chunks +per shard an index would look like: + +.. code-block:: + + | chunk (0, 0) | chunk (0, 1) | chunk (1, 0) | chunk (1, 1) | + | offset | length | offset | length | offset | length | offset | length | + | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | + + +Empty chunks are denoted by setting both offset and length to `2^64 - 1``. +The index always has the full shape of all possible chunks per shard, +even if they are outside of the array size. + +The actual order of the chunk-content is not fixed and may be chosen by the implementation +as all possible write orders are valid according to this specification and therefore can +be read by any other implementation. When writing partial chunks into an existing shard no +specific order of the existing chunks may be expected. Some writing strategies might be + +* **Fixed order**: Specify a fixed order (e.g. row-, column-major or Morton order). + When replacing existing chunks larger or equal sized chunks may be replaced in-place, + leaving unused space up to an upper limit which might possibly be specified. + Please note that for regular-sized uncompressed data all chunks have the same size and + can therefore be replaced in-place. +* **Append-only**: Any chunk to write is appended to the existing shard, followed by an updated index. + +Any configuration parameters for the write strategy must not be part of the metadata document, +in a shard I'd propose to use Morton order, but this can easily be changed and customized, since any order can be read. + + +Key translation +=============== + +The Zarr store interface is defined in terms of `keys` and `values`, +where a `key` is a sequence of characters and a `value` is a sequence +of bytes. + +@@TODO + + +Store API implementation +======================== + +@@TODO + +The section below defines an implementation of the Zarr abstract store +interface (@@TODO link) in terms of the native operations of this +storage system. Below ``fspath_to_key()`` is a function that +translates file system paths to store keys, and ``key_to_fspath()`` is +a function that translates store keys to file system paths, as defined +in the section above. + +* ``get(key) -> value`` : Read and return the contents of the file at + file system path ``key_to_fspath(key)``. + +* ``set(key, value)`` : Write ``value`` as the contents of the file at + file system path ``key_to_fspath(key)``. + +* ``delete(key)`` : Delete the file or directory at file system path + ``key_to_fspath(key)``. + +* ``list()`` : Recursively walk the file system from the base + directory, returning an iterator over keys obtained by calling + ``fspath_to_key(fp)`` for each descendant file path ``fp``. + +* ``list_prefix(prefix)`` : Obtain a file system path by calling + ``key_to_fspath(prefix)``. If the result is a directory path, + recursively walk the file system from this directory, returning an + iterator over keys obtained by calling ``fspath_to_key(fp)`` for + each descendant file path ``fp``. + +* ``list_dir(prefix)`` : Obtain a file system path by calling + ``key_to_fspath(prefix)``. If the result is a director path, list + the directory children. Return a set of keys obtained by calling + ``fspath_to_key(fp)`` for each child file path ``fp``, and a set of + prefixes obtained by calling ``fspath_to_key(dp)`` for each child + directory path ``dp``. + + +References +========== + +.. [RFC2119] S. Bradner. Key words for use in RFCs to Indicate + Requirement Levels. March 1997. Best Current Practice. URL: + https://tools.ietf.org/html/rfc2119 + + +Change log +========== + +@@TODO From 5d7c9532c4cf82f8f26177fa5d5aa9b756b8d05f Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 22 Feb 2022 12:29:44 +0100 Subject: [PATCH 02/12] finished first draft of sharding spec --- docs/protocol/core/v3.0.rst | 21 +++-- docs/storage_transformers/sharding/v1.0.rst | 99 +++++++++------------ 2 files changed, 57 insertions(+), 63 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index da589aab..6242eb35 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -393,7 +393,9 @@ conceptual model underpinning the Zarr protocol. an array_ in the underlying store_. Upon retrival the original data is restored within the transformer. Any number of `predefined storage transformers`_ can be registered and stacked. + See the `storage transformers details`_ below. +.. _`storage transformers details`: #storage-transformers-1 Node names ========== @@ -1034,14 +1036,14 @@ The following names are optional: ``storage_transformers`` - Specifies a codec to be used for encoding and decoding chunks. The - value must be an object containing the name ``codec`` whose value - is a URI that identifies a codec and dereferences to a human-readable - representation of the codec specification. The codec + Specifies a stack of `storage transformers`_. Each value in the list must + be an object containing the name ``storage_transformer`` whose value + is a URI that identifies a storage transformer and dereferences to a + human-readable representation of the codec specification. The object may also contain a ``configuration`` object which consists of the - parameter names and values as defined by the corresponding codec - specification. When the ``compressor`` name is absent, this means that no - compressor is used. + parameter names and values as defined by the corresponding storage transformer + specification. When the ``storage_transformers`` name is absent no storage + transformer is used, same for an empty list. All other names within the array metadata object are reserved for @@ -1197,6 +1199,8 @@ a store implementation to support all of these capabilities. A **readable store** supports the following operation: +@@TODO add bundled & partial access + ``get`` - Retrieve the `value` associated with a given `key`. | Parameters: `key` @@ -1528,7 +1532,8 @@ Storage transformers A Zarr storage transformer allows to change the zarr-compatible data before storing it. The stored transformed data is restored to its original state whenever data is requested -by the Array. +by the Array. Storage transformers can be configured per array via the +``storage_transformers`` name in the `array metadata`_. A storage transformer serves the same `Abstract store interface`_ as the store_. However, it should not persistently store any information necessary to restore the original data, diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 2aa7d487..99838449 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -27,8 +27,8 @@ License `_. Abstract ======== -This specification defines an implementation of the Zarr abstract -storage transformer API introducing sharding. +This specification defines an implementation of the Zarr +storage transformer protocol for sharding. Motivation @@ -69,7 +69,7 @@ this specification are introduced with the words "for example". Configuration ============= -:ref:`array-metadata`. +Sharding can be configured per array in the :ref:`array-metadata`: .. code-block:: @@ -87,11 +87,49 @@ Configuration ] } +``format`` -Sharding Mechanism -========================= + Specifies a `Binary shard format`_. In this version, the only binary format is the + ``indexed`` format. -@@TODO +``chunks_per_shard`` + + An array of integers providing the number of chunks that are combined in a shard + for each dimension of the Zarr array, where each chunk may only start at a position + that is divisble by ``chunks_per_shard`` per dimension, e.g. starting at the zero-origin. + The length of the array must match the length of the array metadata ``shape`` entry. + For example, a value ``[32, 2]`` indicates that 64 chunks are combined in one shard, + 32 along the first dimension, and for each of those 2 along the second dimension. + Valid starting positions for a shard in the chunk-grid are therefore ``[0, 0]``, + ``[32, 2]``, ``[32, 4]``, ``[64, 2]`` or ``[96, 18]``. + + +Key & value transformation +========================== + +The storage transformer protocol defines the abstract interface to be the same +as the `Abstract store interface`_. + +The Zarr store interface is defined in terms of `keys` and `values`, +where a `key` is a sequence of characters and a `value` is a sequence +of bytes. A key-value pair is called entry in the following part. + +This sharding transformer only adapts entries where the key starts +with `data/root`, as they indicate data keys for array chunks. All other +entries are simply passed on. + +Entries starting with `data/root` are grouped by their common shard, assuming +`Storage keys` from a regular chunk grid which may use a customly configured +``chunk separator``: +For all entries that are part of the same shard the key is changed to the +shard-key and the values are combined in the `Binary shard format`_ described +below. The new shard-key is the chunk key divided by ``chunks_per_shard`` and +floored per dimension. E.g. for ``chunks_per_shard=[32, 2]``, the chunk grid +position ``[96, 18]`` (e.g. key "data/root/foo/baz/c96/18") is transformed to +the shard grid position ``[3, 9]`` and reassigned to the respective new key, +honoring the original chunk separator (e.g. "data/root/foo/baz/c3/9"). +Chunk grid positions ``[96, 19]``, ``[97, 18]``, …, up to ``[127, 19]`` will +also have the same shard grid position ``[3, 9]``. Binary shard format @@ -133,55 +171,6 @@ Any configuration parameters for the write strategy must not be part of the meta in a shard I'd propose to use Morton order, but this can easily be changed and customized, since any order can be read. -Key translation -=============== - -The Zarr store interface is defined in terms of `keys` and `values`, -where a `key` is a sequence of characters and a `value` is a sequence -of bytes. - -@@TODO - - -Store API implementation -======================== - -@@TODO - -The section below defines an implementation of the Zarr abstract store -interface (@@TODO link) in terms of the native operations of this -storage system. Below ``fspath_to_key()`` is a function that -translates file system paths to store keys, and ``key_to_fspath()`` is -a function that translates store keys to file system paths, as defined -in the section above. - -* ``get(key) -> value`` : Read and return the contents of the file at - file system path ``key_to_fspath(key)``. - -* ``set(key, value)`` : Write ``value`` as the contents of the file at - file system path ``key_to_fspath(key)``. - -* ``delete(key)`` : Delete the file or directory at file system path - ``key_to_fspath(key)``. - -* ``list()`` : Recursively walk the file system from the base - directory, returning an iterator over keys obtained by calling - ``fspath_to_key(fp)`` for each descendant file path ``fp``. - -* ``list_prefix(prefix)`` : Obtain a file system path by calling - ``key_to_fspath(prefix)``. If the result is a directory path, - recursively walk the file system from this directory, returning an - iterator over keys obtained by calling ``fspath_to_key(fp)`` for - each descendant file path ``fp``. - -* ``list_dir(prefix)`` : Obtain a file system path by calling - ``key_to_fspath(prefix)``. If the result is a director path, list - the directory children. Return a set of keys obtained by calling - ``fspath_to_key(fp)`` for each child file path ``fp``, and a set of - prefixes obtained by calling ``fspath_to_key(dp)`` for each child - directory path ``dp``. - - References ========== From c40725d9b4d56ef1da31d3c894f75ac3f7fee0d0 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 22 Feb 2022 12:53:17 +0100 Subject: [PATCH 03/12] minor fixes --- docs/protocol/core/v3.0.rst | 3 +++ docs/storage_transformers/sharding/v1.0.rst | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index 6242eb35..c857f5e2 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1167,6 +1167,9 @@ interface`_ subsection. The store interface can be implemented using a variety of underlying storage technologies, described in the subsection on `Store implementations`_. + +.. _abstract-store-interface: + Abstract store interface ------------------------ diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 99838449..2c61bfe7 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -108,18 +108,18 @@ Key & value transformation ========================== The storage transformer protocol defines the abstract interface to be the same -as the `Abstract store interface`_. +as the :ref:`abstract-store-interface`. The Zarr store interface is defined in terms of `keys` and `values`, where a `key` is a sequence of characters and a `value` is a sequence -of bytes. A key-value pair is called entry in the following part. +of bytes. A key-value pair is called `entry` in the following part. This sharding transformer only adapts entries where the key starts with `data/root`, as they indicate data keys for array chunks. All other entries are simply passed on. -Entries starting with `data/root` are grouped by their common shard, assuming -`Storage keys` from a regular chunk grid which may use a customly configured +Entries starting with ``data/root`` are grouped by their common shard, assuming +storage keys from a regular chunk grid which may use a customly configured ``chunk separator``: For all entries that are part of the same shard the key is changed to the shard-key and the values are combined in the `Binary shard format`_ described From 7f0b2387eeda15b0dd95d77a69f195024e8ea28c Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Tue, 22 Feb 2022 14:20:20 +0100 Subject: [PATCH 04/12] apply feedback --- docs/storage_transformers/sharding/v1.0.rst | 22 +++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 2c61bfe7..66ace16a 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -104,8 +104,11 @@ Sharding can be configured per array in the :ref:`array-metadata`: ``[32, 2]``, ``[32, 4]``, ``[64, 2]`` or ``[96, 18]``. +Storage transformer implementation +================================== + Key & value transformation -========================== +-------------------------- The storage transformer protocol defines the abstract interface to be the same as the :ref:`abstract-store-interface`. @@ -124,7 +127,7 @@ storage keys from a regular chunk grid which may use a customly configured For all entries that are part of the same shard the key is changed to the shard-key and the values are combined in the `Binary shard format`_ described below. The new shard-key is the chunk key divided by ``chunks_per_shard`` and -floored per dimension. E.g. for ``chunks_per_shard=[32, 2]``, the chunk grid +floored per dimension. For example for ``chunks_per_shard=[32, 2]``, the chunk grid position ``[96, 18]`` (e.g. key "data/root/foo/baz/c96/18") is transformed to the shard grid position ``[3, 9]`` and reassigned to the respective new key, honoring the original chunk separator (e.g. "data/root/foo/baz/c3/9"). @@ -133,16 +136,18 @@ also have the same shard grid position ``[3, 9]``. Binary shard format -=================== +------------------- The only binary format is the ``indexed`` format, as specified by the ``format`` configuration key. Other binary formats might be added in future versions. In the indexed binary format chunks are written successively in a shard, where unused space between them is allowed, followed by an index referencing them. +The index is placed at the end of the file and has a length of 16 bytes per chunk +in a shard, for example ``16 bytes * 64 = 1014 bytes`` for ``chunks_per_shard=[32, 2]``. The index holds an `offset, length` pair of little-endian uint64 per chunk, -the chunks-order in the index is row-major (C) order, e.g. for (2, 2) chunks -per shard an index would look like: +the chunks-order in the index is row-major (C) order, for example for +``chunks_per_shard=[2, 2]`` an index would look like: .. code-block:: @@ -151,7 +156,7 @@ per shard an index would look like: | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | -Empty chunks are denoted by setting both offset and length to `2^64 - 1``. +Empty chunks are denoted by setting both offset and length to ``2^64 - 1``. The index always has the full shape of all possible chunks per shard, even if they are outside of the array size. @@ -165,10 +170,11 @@ specific order of the existing chunks may be expected. Some writing strategies m leaving unused space up to an upper limit which might possibly be specified. Please note that for regular-sized uncompressed data all chunks have the same size and can therefore be replaced in-place. -* **Append-only**: Any chunk to write is appended to the existing shard, followed by an updated index. +* **Append-only**: Any chunk to write is appended to the existing shard, + followed by an updated index. Any configuration parameters for the write strategy must not be part of the metadata document, -in a shard I'd propose to use Morton order, but this can easily be changed and customized, since any order can be read. +they need to be configured at runtime, as this is implementation specific. References From 28a90230a7e56953e164f30bcb32a6afec6f5337 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Wed, 23 Feb 2022 12:57:25 +0100 Subject: [PATCH 05/12] adapt abstract store interface --- docs/protocol/core/v3.0.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index c857f5e2..1d176191 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1191,6 +1191,22 @@ one such pair for any given `key`. I.e., a store is a mapping from keys to values. It is also assumed that keys are case sensitive, i.e., the keys "foo" and "FOO" are different. +To read and write partial values, a `range` specifies two integers +`range_start` and `range_length`, that specify a part of the value +starting at byte `range_start` (inclusive) and having a length of +`range_length` bytes. `range_length` may be none, indicating all +available data until the end of the referenced value. For example +`range` ``[0, none]`` specifies the full value. Stores that do not +support partial access can still answer the requests using cutouts +of full values. It is recommended that the implementation of the +``get_multi``, ``set_multi`` and ``remove_multi`` methods is made +optional, providing fallbacks for them by default. However, it is +recommended to supply those operations where possible for efficiency. +Also, the ``get``, ``set`` and ``erase`` can easily be mapped onto +their `multi` counterparts. Therefore, it is also recommended to +supply fallbacks for those if the `multi` operations can be implemented. +An entity containing those fallbacks could be named ``MultiStore``. + The store interface also defines some operations involving `prefixes`. In the context of this interface, a prefix is a string containing only characters that are valid for use in `keys` and ending @@ -1209,6 +1225,12 @@ A **readable store** supports the following operation: | Parameters: `key` | Output: `value` +``get_multi`` - Retrieve possibly partial `values` from given `key_ranges`. + + | Parameters: `key_ranges`: ordered set of `key`, `range` pairs, + | a `key` may occur multiple times with different `ranges` + | Output: list of `values`, in the order of the `key_ranges` + A **writeable store** supports the following operations: ``set`` - Store a (`key`, `value`) pair. @@ -1216,11 +1238,25 @@ A **writeable store** supports the following operations: | Parameters: `key`, `value` | Output: none +``set_multi`` - Store `values` at a given `key`, starting at byte `range_start`. + + | Parameters: `key_start_values`: set of `key`, + | `range_start`, `value` triples, a `key` may occur multiple + | times with different `range_starts`, `range_starts` with + | length of the respective `value` must not specify overlapping + | ranges for the same `key` + | Output: none + ``erase`` - Erase the given key/value pair from the store. | Parameters: `key` | Output: none +``erase_multi`` - Erase the given key/value pair from the store. + + | Parameters: `keys`: set of `keys` + | Output: none + ``erase_prefix`` - Erase all keys with the given prefix from the store: | Parameter: `prefix` From 15291b21566dd0618dc2fe44ebabdcf4f765ca7e Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 24 Feb 2022 09:34:42 +0100 Subject: [PATCH 06/12] Apply suggestions from code review Co-authored-by: Norman Rzepka --- docs/storage_transformers/sharding/v1.0.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 66ace16a..73955564 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -37,14 +37,11 @@ Motivation Sharding decouples the concept of chunks from storage keys, which become shards. This is helpful when the requirements for those don't align: -- Compressible units of chunks often need to be read and written in smaller - chunks, whereas -- storage often is optimized for larger data per entry and fewer entries, e.g. - as restricted by the file block size and maximum inode number for typical - file systems. +- Chunk sizes need to be small for read efficiency requirements, e.g. for data streaming in browser-based visualization software, whereas +- it becomes inefficient or impractical to store a large number of chunks in single files or objects due to the design constraints of the underlying storage, e.g. as restricted by the file block size and maximum inode number for typical file systems. This does not necessarily fit the access patterns of the data, so chunks might -need to be smaller than one storage key. In those cases sharding decouples those +need to be smaller than the minimum size of one storage key. In those cases sharding decouples those entities. One shard corresponds to one storage key, but can contain multiple chunks: .. image:: sharding.png @@ -113,7 +110,7 @@ Key & value transformation The storage transformer protocol defines the abstract interface to be the same as the :ref:`abstract-store-interface`. -The Zarr store interface is defined in terms of `keys` and `values`, +The Zarr store interface is defined as a mapping of `keys` and `values`, where a `key` is a sequence of characters and a `value` is a sequence of bytes. A key-value pair is called `entry` in the following part. @@ -143,7 +140,7 @@ configuration key. Other binary formats might be added in future versions. In the indexed binary format chunks are written successively in a shard, where unused space between them is allowed, followed by an index referencing them. -The index is placed at the end of the file and has a length of 16 bytes per chunk +The index is placed at the end of the file and has a length of 16 bytes multiplied by the number of chunks in a shard, for example ``16 bytes * 64 = 1014 bytes`` for ``chunks_per_shard=[32, 2]``. The index holds an `offset, length` pair of little-endian uint64 per chunk, the chunks-order in the index is row-major (C) order, for example for From c7c59050db5abd61f7364b5a541c8ed9052d5329 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 24 Feb 2022 09:51:17 +0100 Subject: [PATCH 07/12] rename multi methods to partial_values --- docs/protocol/core/v3.0.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index 1d176191..b85f58c1 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1199,13 +1199,14 @@ available data until the end of the referenced value. For example `range` ``[0, none]`` specifies the full value. Stores that do not support partial access can still answer the requests using cutouts of full values. It is recommended that the implementation of the -``get_multi``, ``set_multi`` and ``remove_multi`` methods is made -optional, providing fallbacks for them by default. However, it is -recommended to supply those operations where possible for efficiency. -Also, the ``get``, ``set`` and ``erase`` can easily be mapped onto -their `multi` counterparts. Therefore, it is also recommended to -supply fallbacks for those if the `multi` operations can be implemented. -An entity containing those fallbacks could be named ``MultiStore``. +``get_partial_values``, ``set_partial_values`` and +``remove_partial_values`` methods is made optional, providing fallbacks +for them by default. However, it is recommended to supply those operations +where possible for efficiency. Also, the ``get``, ``set`` and ``erase`` +can easily be mapped onto their `partial_values` counterparts. +Therefore, it is also recommended to supply fallbacks for those if the +`partial_values` operations can be implemented. +An entity containing those fallbacks could be named ``StoreWithPartialAccess``. The store interface also defines some operations involving `prefixes`. In the context of this interface, a prefix is a string @@ -1225,7 +1226,7 @@ A **readable store** supports the following operation: | Parameters: `key` | Output: `value` -``get_multi`` - Retrieve possibly partial `values` from given `key_ranges`. +``get_partial_values`` - Retrieve possibly partial `values` from given `key_ranges`. | Parameters: `key_ranges`: ordered set of `key`, `range` pairs, | a `key` may occur multiple times with different `ranges` @@ -1238,7 +1239,7 @@ A **writeable store** supports the following operations: | Parameters: `key`, `value` | Output: none -``set_multi`` - Store `values` at a given `key`, starting at byte `range_start`. +``set_partial_values`` - Store `values` at a given `key`, starting at byte `range_start`. | Parameters: `key_start_values`: set of `key`, | `range_start`, `value` triples, a `key` may occur multiple @@ -1252,7 +1253,7 @@ A **writeable store** supports the following operations: | Parameters: `key` | Output: none -``erase_multi`` - Erase the given key/value pair from the store. +``remove_partial_values`` - Erase the given key/value pair from the store. | Parameters: `keys`: set of `keys` | Output: none From 720febb6055160c4bb9df74fee3b0068f1227a25 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 24 Feb 2022 09:54:58 +0100 Subject: [PATCH 08/12] improve motivation, add reference --- docs/protocol/core/v3.0.rst | 2 ++ docs/storage_transformers/sharding/v1.0.rst | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index b85f58c1..5f59b802 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1366,6 +1366,8 @@ Note that any non-root hierarchy path will have ancestor paths that identify ancestor nodes in the hierarchy. For example, the path "/foo/bar" has ancestor paths "/foo" and "/". +.. _storage-keys: + Storage keys ------------ diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 73955564..19c709f6 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -30,19 +30,22 @@ Abstract This specification defines an implementation of the Zarr storage transformer protocol for sharding. +Sharding co-locates multiple chunks within a storage object, bundling them in shards. + Motivation ========== -Sharding decouples the concept of chunks from storage keys, which become shards. -This is helpful when the requirements for those don't align: +In many cases it becomes inefficient or impractical to store a large number of chunks in +single files or objects due to the design constraints of the underlying storage, +for example as restricted by the file block size and maximum inode number for typical file systems. -- Chunk sizes need to be small for read efficiency requirements, e.g. for data streaming in browser-based visualization software, whereas -- it becomes inefficient or impractical to store a large number of chunks in single files or objects due to the design constraints of the underlying storage, e.g. as restricted by the file block size and maximum inode number for typical file systems. +Increasing the chunk size works only up to a certain point, as chunk sizes need to be small for +read efficiency requirements, for example to stream data in browser-based visualization software. -This does not necessarily fit the access patterns of the data, so chunks might -need to be smaller than the minimum size of one storage key. In those cases sharding decouples those -entities. One shard corresponds to one storage key, but can contain multiple chunks: +Therefore, chunks may need to be smaller than the minimum size of one storage key. +In those cases it is required to store objects at a more coarse granularity than reading chunks. +Sharding solves this by allowing to store multiple chunks in one storage key, which is called a shard: .. image:: sharding.png @@ -115,8 +118,8 @@ where a `key` is a sequence of characters and a `value` is a sequence of bytes. A key-value pair is called `entry` in the following part. This sharding transformer only adapts entries where the key starts -with `data/root`, as they indicate data keys for array chunks. All other -entries are simply passed on. +with `data/root`, as they indicate data keys for array chunks, see +:ref:`storage-keys`. All other entries are simply passed on. Entries starting with ``data/root`` are grouped by their common shard, assuming storage keys from a regular chunk grid which may use a customly configured From e98cbc02b02e02385276445e1fd04b6f8a3ef188 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 24 Feb 2022 12:17:48 +0100 Subject: [PATCH 09/12] add api implementation protocol --- docs/protocol/core/v3.0.rst | 7 +- docs/storage_transformers/sharding/v1.0.rst | 88 +++++++++++++++++++++ docs/stores/filesystem/v1.0.rst | 4 +- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index 5f59b802..9698d30c 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1200,7 +1200,7 @@ available data until the end of the referenced value. For example support partial access can still answer the requests using cutouts of full values. It is recommended that the implementation of the ``get_partial_values``, ``set_partial_values`` and -``remove_partial_values`` methods is made optional, providing fallbacks +``erase_values`` methods is made optional, providing fallbacks for them by default. However, it is recommended to supply those operations where possible for efficiency. Also, the ``get``, ``set`` and ``erase`` can easily be mapped onto their `partial_values` counterparts. @@ -1230,7 +1230,8 @@ A **readable store** supports the following operation: | Parameters: `key_ranges`: ordered set of `key`, `range` pairs, | a `key` may occur multiple times with different `ranges` - | Output: list of `values`, in the order of the `key_ranges` + | Output: list of `values`, in the order of the `key_ranges`, may contain none + | for missing keys A **writeable store** supports the following operations: @@ -1253,7 +1254,7 @@ A **writeable store** supports the following operations: | Parameters: `key` | Output: none -``remove_partial_values`` - Erase the given key/value pair from the store. +``erase_values`` - Erase the given key/value pairs from the store. | Parameters: `keys`: set of `keys` | Output: none diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 19c709f6..c507bca4 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -177,6 +177,94 @@ Any configuration parameters for the write strategy must not be part of the meta they need to be configured at runtime, as this is implementation specific. +API implementation +------------------ + +The section below defines an implementation of the +:ref:`abstract-store-interface` in terms of the operations of this +storage transformer as a ``StoreWithPartialAccess``. +The term `underlying store` references either the next storage transformer +in the stack or the actual store if this transformer is the last one in the +stack. Any operations with keys not starting with ``data/root`` are simply +relayed to the underlying store and not described explicitly. + +* ``get_partial_values(key_ranges) -> values``: + For each referenced key, request the indices from the underlying store using + ``get_partial_values``. For each `key`, `range` pair in in `key_ranges`, + check if the chunk exists by checking if the index offset and length + are both ``2^64 - 1``. For existing keys, request the actual chunks by + their ranges as read from the index using ``get_partial_values``. + This operation should be implemented using two ``get_partial_values`` + operations on the underlying store, one for retrieving the indices and + one for retrieving existing chunks. + +* ``set_partial_values(key_start_values)`` : + For each referenced key, check if all available chunks in a shard are + referenced. In this case a shard can be constructed according to the + `Binary shard format`_ directly. + For all other keys, request the indices from the underlying store using + ``get_partial_values``. All chunks that are not updated completely and + exist according to the index (index offset and length are both + ``2^64 - 1``) need to be read via ``get_partial_values`` from the + underlying store. For simplification purposes a shard may also be read + completely, combining the previous two `get` operations into one. + Based on the existing chunks and value ranges that need to be updated + new shards are constructed according to the `Binary shard format`_. + All shards that need to be updated must now be set via ``set`` or + ``set_partial_values(key_start_values)``, depending one the chosen + writing strategy provided by the implementation. + Specialized store implementations that allow appending to a storage + object may only need to read the index to update it. + +* ``erase_values(keys)`` : + For each referenced key, check if all available chunks in a shard are + referenced. In this case the full shard is removed using ``erase_values`` + on the underlying store. + For all other keys, request the indices from the underlying + store using ``get_partial_values``. Update the index using and offset and + length of ``2^64 - 1`` to mark missing chunks. The updated index may be + be written in-place using ``set_partial_values(key_start_values)``, + or a larger rewrite of the shard may be done including the index update, + but also removing value ranges corresponding to the erased chunks. + +* ``erase_prefix()`` : If the prefix contains a part of the chunk-grid + key, this part is translated to the referenced shard and contained chunks. + For affected shards where all contained chunks are erased the prefix is + rewritten to the corresponding shard key and the operation is relayed to + the underlying store. + For all shards where only some chunks are erased the affected chunks + are removed by invoking the operation ``erase_values`` on this + storage transformer with the respective chunk keys. + +* ``list()``: See ``list_prefix`` with the prefix ``/``. + +* ``list_prefix(prefix)`` : If the prefix contains a part of the chunk-grid + key, this part is translated to the referenced shard and contained chunks. + Then, ``list_prefix`` is called on the underlying store with the translated + prefix. For all listed shards request the indices from the underlying store + using ``get_partial_values``. Existing chunks, where the index offset or + length are not ``2^64 - 1`` are then listed by their original key. + +* ``list_dir(prefix)`` : If the prefix contains a part of the chunk-grid + key, this part is translated to the referenced shard and contained chunks. + Then, ``list_dir`` is called on the underlying store with the translated + prefix. For all *retrieved prefixes* (not full keys) with partial shard keys, + the corresponding original prefixes covering all possible chunks in the shard + are listed. For *retrieved full keys* the the indices from the underlying store + are requested using ``get_partial_values``. Existing chunks, where the index + offset or length are not ``2^64 - 1`` are then listed by their original key. + + .. note:: + Not all listed prefixes must necessarily contain keys, as shard prefixes with + partially available chunks return prefixes for all possible chunks without + verifying their exisence for performance reasons. Listing those prefixes + is still safe as some chunks in their corresponding shard exist, but not + necessarily in the requested prefix, possibly leading to empty responses. + Please note, this only applies for returned prefixes, *not* for full keys + referencing storage objects. Returned full keys always reflect the actually + available chunks and are safe to request. + + References ========== diff --git a/docs/stores/filesystem/v1.0.rst b/docs/stores/filesystem/v1.0.rst index 79e303c4..df070709 100644 --- a/docs/stores/filesystem/v1.0.rst +++ b/docs/stores/filesystem/v1.0.rst @@ -149,8 +149,8 @@ directory path is "C:\\data", then the file system path Store API implementation ======================== -The section below defines an implementation of the Zarr abstract store -interface (@@TODO link) in terms of the native operations of this +The section below defines an implementation of the Zarr +:ref:`abstract-store-interface` in terms of the native operations of this storage system. Below ``fspath_to_key()`` is a function that translates file system paths to store keys, and ``key_to_fspath()`` is a function that translates store keys to file system paths, as defined From c232b05870a09ab9adde71d3831bc66a73ac8fe5 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Thu, 24 Feb 2022 14:49:37 +0100 Subject: [PATCH 10/12] Update docs/storage_transformers/sharding/v1.0.rst Co-authored-by: Norman Rzepka --- docs/storage_transformers/sharding/v1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index c507bca4..338518a8 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -44,7 +44,7 @@ Increasing the chunk size works only up to a certain point, as chunk sizes need read efficiency requirements, for example to stream data in browser-based visualization software. Therefore, chunks may need to be smaller than the minimum size of one storage key. -In those cases it is required to store objects at a more coarse granularity than reading chunks. +In those cases it is efficient to store objects at a more coarse granularity than reading chunks. Sharding solves this by allowing to store multiple chunks in one storage key, which is called a shard: .. image:: sharding.png From ba834f32297e4c1dc0c3dce25a9a06dbbbd3b3aa Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Wed, 20 Apr 2022 15:16:43 +0200 Subject: [PATCH 11/12] use nbytes instead of length in the index description --- docs/storage_transformers/sharding/v1.0.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/storage_transformers/sharding/v1.0.rst b/docs/storage_transformers/sharding/v1.0.rst index 338518a8..66c7fe98 100644 --- a/docs/storage_transformers/sharding/v1.0.rst +++ b/docs/storage_transformers/sharding/v1.0.rst @@ -143,20 +143,20 @@ configuration key. Other binary formats might be added in future versions. In the indexed binary format chunks are written successively in a shard, where unused space between them is allowed, followed by an index referencing them. -The index is placed at the end of the file and has a length of 16 bytes multiplied by the number of chunks +The index is placed at the end of the file and has a size of 16 bytes multiplied by the number of chunks in a shard, for example ``16 bytes * 64 = 1014 bytes`` for ``chunks_per_shard=[32, 2]``. -The index holds an `offset, length` pair of little-endian uint64 per chunk, +The index holds an `offset, nbytes` pair of little-endian uint64 per chunk, the chunks-order in the index is row-major (C) order, for example for ``chunks_per_shard=[2, 2]`` an index would look like: .. code-block:: | chunk (0, 0) | chunk (0, 1) | chunk (1, 0) | chunk (1, 1) | - | offset | length | offset | length | offset | length | offset | length | + | offset | nbytes | offset | nbytes | offset | nbytes | offset | nbytes | | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | uint64 | -Empty chunks are denoted by setting both offset and length to ``2^64 - 1``. +Empty chunks are denoted by setting both offset and nbytes to ``2^64 - 1``. The index always has the full shape of all possible chunks per shard, even if they are outside of the array size. @@ -191,7 +191,7 @@ relayed to the underlying store and not described explicitly. * ``get_partial_values(key_ranges) -> values``: For each referenced key, request the indices from the underlying store using ``get_partial_values``. For each `key`, `range` pair in in `key_ranges`, - check if the chunk exists by checking if the index offset and length + check if the chunk exists by checking if the index offset and nbytes are both ``2^64 - 1``. For existing keys, request the actual chunks by their ranges as read from the index using ``get_partial_values``. This operation should be implemented using two ``get_partial_values`` @@ -204,7 +204,7 @@ relayed to the underlying store and not described explicitly. `Binary shard format`_ directly. For all other keys, request the indices from the underlying store using ``get_partial_values``. All chunks that are not updated completely and - exist according to the index (index offset and length are both + exist according to the index (index offset and nbytes are both ``2^64 - 1``) need to be read via ``get_partial_values`` from the underlying store. For simplification purposes a shard may also be read completely, combining the previous two `get` operations into one. @@ -222,7 +222,7 @@ relayed to the underlying store and not described explicitly. on the underlying store. For all other keys, request the indices from the underlying store using ``get_partial_values``. Update the index using and offset and - length of ``2^64 - 1`` to mark missing chunks. The updated index may be + nbytes of ``2^64 - 1`` to mark missing chunks. The updated index may be be written in-place using ``set_partial_values(key_start_values)``, or a larger rewrite of the shard may be done including the index update, but also removing value ranges corresponding to the erased chunks. @@ -243,7 +243,7 @@ relayed to the underlying store and not described explicitly. Then, ``list_prefix`` is called on the underlying store with the translated prefix. For all listed shards request the indices from the underlying store using ``get_partial_values``. Existing chunks, where the index offset or - length are not ``2^64 - 1`` are then listed by their original key. + nbytes are not ``2^64 - 1`` are then listed by their original key. * ``list_dir(prefix)`` : If the prefix contains a part of the chunk-grid key, this part is translated to the referenced shard and contained chunks. @@ -252,7 +252,7 @@ relayed to the underlying store and not described explicitly. the corresponding original prefixes covering all possible chunks in the shard are listed. For *retrieved full keys* the the indices from the underlying store are requested using ``get_partial_values``. Existing chunks, where the index - offset or length are not ``2^64 - 1`` are then listed by their original key. + offset or nbytes are not ``2^64 - 1`` are then listed by their original key. .. note:: Not all listed prefixes must necessarily contain keys, as shard prefixes with From 75d5e49bc46a16c4db08e156d5133a0fc0487e78 Mon Sep 17 00:00:00 2001 From: Jonathan Striebel Date: Wed, 20 Apr 2022 15:21:14 +0200 Subject: [PATCH 12/12] add a note about non-transforming transformers (e.g. caches) --- docs/protocol/core/v3.0.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/protocol/core/v3.0.rst b/docs/protocol/core/v3.0.rst index 9698d30c..5e4cf7df 100644 --- a/docs/protocol/core/v3.0.rst +++ b/docs/protocol/core/v3.0.rst @@ -1576,7 +1576,9 @@ Storage transformers A Zarr storage transformer allows to change the zarr-compatible data before storing it. The stored transformed data is restored to its original state whenever data is requested by the Array. Storage transformers can be configured per array via the -``storage_transformers`` name in the `array metadata`_. +``storage_transformers`` name in the `array metadata`_. Storage transformers which do +not change the storage layout (e.g. for caching) may be specified at runtime without +adding them to the array metadata. A storage transformer serves the same `Abstract store interface`_ as the store_. However, it should not persistently store any information necessary to restore the original data, @@ -1585,7 +1587,7 @@ From the perspective of an Array or a previous stage transformer both store and protocol and can be interchanged regarding the protocol. The behaviour can still be different, e.g. requests may be cached or the form of the underlying data can change. -Storage Transformers may be stacked to combine different functionalities: +Storage transformers may be stacked to combine different functionalities: .. mermaid::