diff --git a/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/README.md b/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/README.md new file mode 100644 index 0000000000..7f4880bd9f --- /dev/null +++ b/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/README.md @@ -0,0 +1,65 @@ +# Record: SP8192 + PE + MIN_LR + SmearGate + AttnOutGate + 4ep TTT — val_bpb 1.0770 (3-seed mean) + +**val_bpb = 1.0770** (3-seed mean, std 0.0004) | **~15.98 MB** | 8xH100 SXM + +## 3-Seed Results + +| Seed | Steps | Sliding BPB | **TTT BPB** | Artifact (bytes) | +|------|-------|-------------|-------------|-------------------| +| 1337 | 4631 | 1.0785 | **1.0772** | 15,982,989 | +| 42 | 4637 | 1.0777 | **1.0765** | 15,984,317 | +| 2024 | 4633 | 1.0784 | **1.0772** | 15,985,404 | +| **Mean** | **4634** | **1.0782** | **1.0770** | **15,984,237** | +| **Std** | | 0.0004 | **0.0004** | | + +Delta vs previous SOTA (1.0783): **-0.0013 BPB** + +## Changes from previous SOTA (2026-04-12) + +### Training improvements +- **Polar Express NS coefficients** — 5 per-iteration minimax-optimal tuples + row normalization (was: fixed 3.4445/-4.775/2.0315) +- **MIN_LR=0.10** warmdown floor (was: 0.0 — LR dropped to zero) +- **QK_GAIN_INIT=5.25** (was: 5.0) +- **GPTQ_RESERVE_SECONDS=0.5** (was: 12.0) +- **VAL_LOSS_EVERY=0** — skip periodic val during training + +### Architecture additions +- **SmearGate** — causal content-gated residual, zero-init transparent +- **Attention Output Gate** — per-head sigmoid gate on attn output (width=12), zero-init + +### TTT improvement +- **4 epochs** (was: 3) of score-first SGD TTT + +## Architecture (unchanged from base) + +``` +SP8192 tokenizer, 11 physical / 17 virtual layers +512 dim, MLP 4x (2048 hidden), GQA 8Q/4KV, head_dim=64 +Parallel residuals L7+, QK-Gain 5.25, XSA all 11 layers +LeakyReLU(0.5)², skip gates, logit softcap 30 +MuonEq-R (lr=0.022, wd=0.095, momentum=0.97) + AdamW +EMA 0.997, warmdown 66.7%, loop at 35% +SDClip GPTQ int6 (k=12.85) + int8 embed (k=20) + brotli +Score-first TTT: SGD lr=0.01, mom=0.9, 4ep, 32K chunks +Hash embedding: 16384x512, zero-init, trained in TTT +~36M params, ~15.98MB artifact +``` + +## Compliance (Track B — Score-First TTT) + +Per Issue #1017: +- **Condition 1:** Hash key uses prefix tokens only +- **Condition 2:** Full normalized softmax distribution +- **Condition 3:** Each chunk scored under no_grad() before TTT update +- **Condition 4:** Single left-to-right pass, no rescoring + +No SLOT, no pre-quant TTT, no n-gram caches, no CaseOps, no global TTT, no multi-phase. + +## Reproduction + +```bash +pip install brotli sentencepiece +MATCHED_FINEWEB_REPO_ID=kevclark/parameter-golf python3 data/cached_challenge_fineweb.py --variant sp8192 --train-shards 80 +SEED=1337 TTT_ENABLED=1 HASH_EMBED_ENABLED=1 TTT_LR=0.01 TTT_EPOCHS=4 TTT_OPTIMIZER=sgd MUON_MOMENTUM=0.97 GLOBAL_TTT_ENABLED=0 \ + torchrun --standalone --nproc_per_node=8 train_gpt.py +``` diff --git a/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/train_gpt.py b/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/train_gpt.py new file mode 100644 index 0000000000..6bbe54b354 --- /dev/null +++ b/records/track_10min_16mb/2026-04-26_V2_PE_MinLR_AttnGate/train_gpt.py @@ -0,0 +1,5 @@ +import lzma as L,base64 as B,linecache as C +S=L.decompress(B.b85decode('{Wp48S^xk9=GL@E0stWa8~^|S5YJf5;S-`oc3l88n@VT6Qap3bt~@<3h>ok~)Km^%m4{rOaiQa2(_3zS%kEP3*8F9$Y#f-ZSHsOI0J>T;4{pdFFBjnfYv-o7{^Aj5A;P`z&C6&<=lqU+_4@b=R3%dnFVxv`=Mxc$Xs5zzo9Ric6{F|ATO1RDSva^{lbrs1l?`BLg7X=K(Q;6QVBe>>XcJ1BumAA2c%17A9=Ndxtz>P;Qj8GzvXz~UFKA&VZOufyRgkT(@mOE^`mG_Q^|@KlW_VERX8baH?N3EM-_&P%L_EFTm43EHf-VQXAZ>}m{0HbiJdo*$GnEANkBhMepqyPcx3Fe`cguZa~?mFyAO7xx@pyp9u#SgbW+7#3_|0HHzND*Q91f>^;{89_J{Mn(ifsT;O=nQfn#Ij{*MK%J|f`^>lsxUub!E!Sa7u`{MQ6F9EpM`M3LtLPR@0)h4vzK5MC|1-JXholX8)ey)dQyK9Qb%sbNCH*vNp0k<6<%6l+hRO0vpF^o+^Y#^WM9+l@#vx0FlzSmKyQ#D?TgIPQ3$mo~ESH~#){o=pZ>bx|DpULrF#8zQk&^auf9^{$UlamN3-&TUdJlF%9!C7`Nwr+MCh4tYsw(vrIp6F_fi?;BNr?Yjf9NFRsW@;Uw#>_q%@dXd)o8ZOKhsZqb@w3rbC(;tlItqrqozqGJ43StQ$S=7bGQ}@c?@-$H5}FVv__HFkSqWth_C&p>@S1ZdEnw#`Ivzv>R)$n%JiuE>_KhkJSnCNR9vU7cn1dmYI&yutlAMSAB?;wu7Q>+c-|I&aO!1l%k8=cGLbfYzwN<8tU(zgQ=&~Eo^92?QzYSl@#x68g1v1y<>AQsxiR?Dwx|Ect3Fd3lX2OtJRoK)+G2QNeT45iNt?soi?z>TV5qPXXL0bf-Bd>!sDt@A{JmOoaQ<02tlEOxUT$8eJxmp#-9IQ)xwK$XbZCknWml?FkJPey3Oh!HCSJ0pw!f{dHJ35hMyGd-5p%Ao~TL(uh6&j*nPzpM~D_Pi#RGM7K`RWlrB_{zIhf|APS=nmH1I(nhX*0=L@J;uhAp_QlY(yb-dgE6_y|7Rp>q7c@kEqR))Je=2+0dGa?Gqlihbzt8Mf#1-`GocpGMXoTsyiuyp}p1S1ZC#)GKcY{`RKa9%Kks9j-!QcP1I7L{1*4?7`3=Lgy%LZ`>=J5I|wT&-gJU7c!PDqxcETv=*tcL0!C(6$R>1N};GOv)h{>3-E)8hWWJT1S8u|z_^dAGufpZd2?JE0io@$D00GS>Pg@t_o^)Xf!Aw)C;MR~Erxq4vF`ao|Wi9v>O}IU;HN-0!x`=h5^(brJI7z-|h?kI%89xtO1G9{Yv6pmqo~IEY#vIr3prx!GCC{BR)3W;E=AZWMu}29J&I)=eyzT4Qj^<%E{Gjsz-IPk2zFXL}PlgK?lTLUDf2xJV%jd<}G5{7;s%MI*d-E}vc<81vTbKU8Ovi4wKJ0SzfHv|8m3(68{y_9ZjVmI|7;ikL^5G}Y4Ii>rA4uchD-)Ek!hh_wdv?q)C8oK)a=YNvR0xr{t7A?V0$BP#Zfikn5!vm$TeAiv?8owPt^#r520z%+TeDu~^C#~grn{U+wf>BG4ZXJG{1R0W2ti7mN9@B(vOt{AM87YIe6DF+n8Y@masb!tAtDx`<$i_QzZ9lz+GIW+&DZm>3_}~x-p!qoq|3tdw4jCRbj^^d!<^4HYG6m!20g4d!Qm4O9n4cKTRnqQX)p$2((L<0VHqMB1KM^d{3QKQokrMG=1{JkIcBTD91$##-fIb&?0)OxLoS2>7k04B^>Cuwxb;P@I0VvV3|+td?YCUXjzPD5lFq6H;FW0QQHb{;_5X2BM5(&rYNUWx7Cdu5<7l-yIkpV29PubxPWgB9NRObTecd5uckQsr6;9n$uR(32C}_sNadJ~G(Ol4Hqv!(?G1dF{1Jo}MDyF}wbU&4P3QPAAvzEfGGGOARa)hM)<1l|#QRcRo9VIjcG@l02nH$b8O)w59)|S4UH=54eO>_dT4@L{g)5ORjNmF~yt7!sUdIYubl{10EoS_#g#Y2=dS6O@j%d=Rp~@sVA{*=Qcr-UUFZQof0G^)fbCBM~3mM)$z;+1)L}6`f`px^On}uQh2eqfUA-nG09L!i(adlrzB_B;O@Fm%Dd}Y2_3^%@b-H@&aL!V#V2D=%)JjC-sO};VzGNP3aKy?b6I|0(fYRLamnJqQg~%C8#aJMW8<&zDu1a1Uk%k04Hbzo?qx8xmZb6FT>Sx2G1A$WG%bmW_jrJfP0!`Se`_~Jve?|H;Ub{p2v~_6$Kl0w-2nEHa(!?@^Y+5`xp$_?hJ9++O5%f|_JKi>eAv7nw5~MsMOeukH3=m~?QEZAZ87#F`axg~+bG_+k614B^MzS|@H|!p5w)7R)b|1?y)Sku7{Z3y{jLBU0QGjuy}(%=i!WM{ASDynYS=Dk4CGGEv-UB6_?_RI%sp*PagwZ7RC><`!PfvEPAqh6loL}Bz6*R^%H&xagBbG=4Gm}~=<%PmeRfdSf-n&VyG}xu88Z%a+%gUBW8Q6dB?UcdC_y6tYnYxy5vk#xm(kSFLTssmuN?pynEQWXuJ7{qOVSC$G5v`EXo)O}xD&}+=RwQ8tGXQDvNha=U?%xHBUbv>_1Q;Q@wbLHdJ&|CTrG&frXTBjrL4i%1}EsdbM8*fIpY1=>5EAn|0s91K$&$H997z%xUlk7t2?@|urh}KW?Mmw!e1Wu*i>5p57n@DW|My^w7=lwY^T~m${YN4i7=o_++#~DLSF}VpVI6`mC+P>IVpyjl-6bn8wI*&4u95a)&kr{&DA9+0pPIO65wTt8!@#Xk7qM!9k5fyUKS#~JwmdKf5QyYnZS227iufIys+KkJ#B24l`yFjmeF-X2y+<9M{(u>gfx|C&(^3f$(H@tTx4&q($P#=r`cf-xYDBT*YvunU8G827rPnpi^r#5auaArXMAI3lr>{_M&D@!a{(+yj8BpKC33xP=7*w-j~3n0=Bg$$*EMLqC{?L+?s~qMv+wRP{!i+Y&y~Si^U264)lbvdD~VzJ&OcaD2_|ukw)h;^-7-7m#!SNyf{_h(Pi)R)Lu((`l_YW5}@0Ii^M|2?);rJX&^Wc7U9@Bz^q2y_?lWE9O5s!zHsK2Qo4(Y0UyxO304{oUA7(H-esj#eS6dt&4io_laNd(X=rAVQv_W(hxrZW_&YR0Hn5BArf)qASGnr_C#gcf3Cd(2&rIo$5zf6sqU8N?8(Yu}5L>PyU4y#zcNN$TZ%1M-Q*{xef&?w;nn50x)3*zP5Ej-Lg^J$*~B!`j}KrmfX)PWL)R**@i2-xmgeU;2Wa3OTlt#+_Exnfq?b?`P+9iNH;8K$H9Pz)*y-xW-c5oVTD)grPG0K3YxyL%axNTyA$h$$|~0|PR!JFY1rqDFt5vm9v$aw{<5IL>7o{i*c13PHLLUwzqd#_a9I_4V}K9b_~>H9bb%!Q*V&!5guT3r02i3VlalF8|v*IjV`~`qq*mn94r-kvirMRG0D%Ul*h|H%*|97!&8uhmxgkRk@Q0ikf&hOAF(R14hjF4V;G;Im8$3<8XyW$8znGm5MN9BBGGa~ms3mvt>GE(<<8=8$W{bCr9~lTKw5K04foPI4^#>Ib-RvW!lyv?x)ai7%oxw@e0GVlPa-D1p>^DiYB<^lV{3pU&v6?o}s997$fSiM?@kCj2J1g73i%wxj3Dsf}F1PwpCkn8px)p~o>lXg9J5K47QcbYH>K5mI%dOe^<#$4`_hKTmyIMDCJQCdIL23SUSrW*Zonil)c}2Y`{K0sZ#dr-a7Xu9otX8?P;27zqgK>dP<;MqBxN`a9liHRV+lzqF+!QGg7|p$xv+@$c1P3|UaAsH{?KVylAkVRX+Apm6x5KqL>N{^x%s5*9G~MY#S|hR6K$19)~nlM2J4?95i@cf;CU`5N+pRcUkc7nk4?rK|mQQis~z|6y$96`{;@dj?pQS!FEW7-{!ZJ3dEWrVFm+D9nj=cZ{2O>4aVRb{yR^&DMzJkq+hSRC2g@@&6sKwewx8M{LNK|nnzp(1P5$_X`iOYCB;^-OVqe8bST9ITzOQ}qoDx%8(Gy_)|CAn3lxD9Y}j&RUBB5PCb971mDtw)|D?gARh(NyD7F>*RCbN!;0lyWAv1B_e(1hH+m*gsWV^21O<$PAzJYrU6Rfeqn#y?Y_xkL&%9*<2`~%71zx48vr_^qbQnhj@ZC|W=elHwufGPLymt4DYh`Q>6Ilj+($h@{=?G$rT^j;*fIqW5KnIPPGQWGI5%PCq*O#W7+6@JLEO;##%9+2nOtJ5jS)@_U1{fzGhrY2D$|vLyd4#R%4?d9CXbe6fom4O!+F-Q?_$#VMpMl-3?}F6%H!e8-OPNS{wn|?w_S$7G1yRgxATk{BcdCXt#w`cDjvZ(HPobWJ2f4~yrg)&629zY_xn^5@TscnPyJlp?10aof&rgFb0k5hmjw2gzyB?g+tVjo_^7HnM57MMqEExOqkup59BrNhI?GUUwa#CnBBR53Y<08A5ZO{8GO(6KDwvg72>4b&z1#~H?*R&vIz8v6U2bJJ2=GuSoF(T<_#NrgHl-ZP@m|s7$RcXh4v5>aLD_WfIP_hk2!ZtG5g3aY%L^ppdJ{HoMV{M+=vJ)CmX;lf7C+iXo=b{qM1{Tt9b`J3{1YL?rCgZsO(%f&_gY&tg*dN>6B?##DJ0dpeT*$JAXt)89aUj2?LYba$Gc?}!xP{dN#_6!Xj-J7s6tzyQZ6Ogt1zlO+F+9fQ^Xd=R@cUj+Qhk0|pE{)c@TZA1f8P5ui-Zf3-Ku-(v4v>CWh_blDirDUqi{j{C})JEJhAdXMLyuxy?Z2A^hVlL4J8jSa8*SaU9m{PE%3x2g{8I&zCrR&{&E2y77oo`t{u3k-IYgAo?4@1U{5VK;Y~o4*GrXoP>#WXnq;>j&CJpz#^^U8ThOTdk@Ycq_Yhow@22C=N6dh;!+IH={ARArAeswy<=EhVcL=ieV~j7oop{g!bK7)OQ%iJ3P2GuQqGGk_of}%mmeFRSu^D$)S-daPuZ{1bEP#h3MD5cTX{B78d1dwq0Cf$jI1UV>{~&Fe0YHL-T+@LX{ft*=0DAhNxJ_3n9A!1cjftWQ8)$uI{j-<3W7B>@S3NO&(7ZEKVkUU4p%;>WOEdN>d+R4)IF(90Ra$oIWMy1Suh?xl&UayGI)d|m`42r)C@!BT@BD-;y-$w)_(BEBMuVW2c?i)G_MF(v>i)KN%^5lH2gC!cV%aui2bM=-RMvHrP6N4C?al;h&~878VNB_{U%^)&>Vv0Vz82y46G5hj?@4L%s4Z${_X3(!(owQ2(CZ^qxcH5cbP*Po=fb=ux^7EHF2W*5aC@?Gg)0F_p?(bTFE+%Q&B8g>9FB(2A^zn(Yj_wzFnx6BGyhMHF-I;XVtDG5Owd8;z(w*7y0oJk_>-vHDyzLR%4_{lD^7AJGmyx=w^>i~dY)tG(O1H$TzLsPq2;Tt!7H?Ln7G&Jh)GiIdq6FtySFUfxxyRn9-L_pFUX(QH>!1im@%JXM6IycHGdKDJ_utES8Kjk(@lZgPM{ob<@6ryl32MVK5z6@UHXjX3RD9-F&uRpq{mfe^kcYE#bowmJGc6^$i4nIk>@IK<(M~Vxujg{0GU`z>2h>l^eniacO=^P--Xth==I2wKO$rTug)h`U{L!puKnpS^4rhTFyRJZm@mdB|jES_X1?I6ULd+|9Q9m^yrcWdi*i_78CoNim0Nw|LJgo^eH3$wlZ<6ExBo&wZ@D^29kBG&$>qLbXO?*m`O=H~~i*5q6YY1cn6*joGYSh$fLe*a@6+6LTchj$_AzGp77EPN-C`Vbc)t0q4-ucZ78K{A@Lp!tIyC2rIw2~+)|Ln8ZzJ)jt3)UzTXz~40vX2eNm3`5YNTy%rR`ttkasEaKAp|b&yXC+p2BIwOb3QQpoL7{a(UWXr-dm3>WnaBghCN*kxoMow~c0cPmUU1b7D0T=mJ`6l4)VLP|Nki>4fr?U^SO=IyZWItc^rTYrx;BOddVx!1m0AEkgHT{%0hObD6hk0XO9$QoZ)Y)IEf{BAG99x`1jPI^BgO$Gvp!6)ZJA4C{t}{^7{^vM+iFth9-tT1xBXE6~Lim-J_wJ+D!|f4LWL*x>{8U(Z8FPI{9f=C!hY?xWn}KP6HVZv`*JsQI8hN)&9wBa<65-3awgX8@g8uDUe(UOmbK+;DoYrQ$CPZF-MakIJCee(rvw7fY})?pQwPe$9DcbYC*wc`MTLrpw|~*UpEh}!|XxlgNbo_goW58M1YuGaRW5t#dgpvrrjDXti0>6X7wGrTwnSqeGNK5{#u^A30U3JcOByF7tF8pKwSMNgX2WuV{XczzD7>o&Qz4F&F7x!Oi9|DgAip$QB%x|icX2xm^??7kb?Ik7!uEHI;s4+maVZqJJNw1|mSW<^;XS<_IqRNTwWZnxfk7%h25Y+Ey*-tHO9yL^A#tV+m_0I1L^1+IPM7%M2QFMW<f<{>l`O@nMcv{oNGlakIm<$T;E-AX2F#*1&zw@9Ov+4OjNxQ!w4t_$p?lCQxY~9?7uaxW%$7MNUi|B%i))w_`Bud9>q*Wr$k}v~W^)6mpX6dQ=`6X?Un%2az$HTGa$%@mowm50f|Vp$>k^u|_gu&f(&}W`lzwa8=?0F*-N6G{upuPY-0?mi)T~s2CPO!O*GD+?G9yuWE=EQI$-hHV}?#w49DPC4Fi7cs7~74vm0?(%Wfs%%%aN9oJ&VspEZ|Ps{T5e|~mVDdsa@RGA{o6x>~2Ahlx5IjFLC*KoKyh|$^{P8q4ymfgyX20t<(U}j47Y&`^@h_0_4pa%!FL!gsEL!Q_c9{1yQ(!ZI<+;`jwez3vWcB;wten(XKie)Lqt!$@pot1xInf)vhSl?k2p;LnO_K(BE0rzs9t;Z|_hy+>FGOSjQ$5Mu|w5n>qv3IHBQ0{yMRD7sx#m#*eXTWgc%R3UX1Gkimy$slK5ZnWlN5s@~t1%G&>g_b;?nBh$qU;>cDM91X-)RiL_^nYQkBss=m;zkKn|;^q5Y&KAN}H!e!?-ILNZYI-i7anG*0tM9d3YKtnG1iB@uLGLt1&&6~JSk9OD~#N|wY;U}Y<7LdZE+WUcMVt4!k~mG++`KMr2Z$~N;nt$Nd3QB^8>5^kLF76h2&@N5=13tH-+B(X(tV+@t)0AxVGkLpEZdz0YehmR9)^vmdg3mHUB9j4oInT;H_-Pt!bCZIzxeY?Lb*D}1{W?ZOpZvILK2lcj5=m28YgWc#E2A0Goo)nVu@wvN)Ham0t*ohQCGw>~&j+G$6ZQKnuX87=-<-ta^%V%BrU9$t_tha8<+SaGx|_Bp(&3ja~$Il7!u0e9EEs)4g5g~D$t{=CeYin>x(HHbdDiYQtoV;Jkg3+PNt^t-AJ&5Rig-{lIRUgUpHM_3Qqq1mFF_4(yxUUk#f{o0DlpbH$6>$0r8RT-Bq5V>CqfZ-iwcvm%W<+l4pPLTb3=qOVo8{=P^7h?+DgULk2iRjm09zVN7FeT&WW(Z5G=ttoDMV+y20&U{T|InT|?xjD*X@`vZuA9YbEVE}`B#{w(k1zn33?#*ICdg%NQ8oTm2+^h);FXPQtzz+k*}V~2|JH@Nu;u$So(G5>a_w)vMSr$-@Nc2|>BP8Y4d+Iy$kepo+B%3z!#))`*x%=+$TF3Ln>09WTe+#-jYEeN?{K}1>4$V1oDO=(w6#qXzRY0tcEsP7wm9GiMnmH!TV)C34vy?$il2@cFZ-3t>TD?c(nqd#ES6criM)Qe9js$Z{?G(-@fJag32zkWO;G&_b%V{*zUlJ$T5Vt`UHVDTm9JTc(wX^yN^AefszaA=@Yur{I-@vvcvPZcn%>e&mXc?%gp-jHdcPK0;vAyeAX}8T11#$5NP!lD_zm>sSbB0BC1XD9x9IkUc}Dzq^LmI;eCXO@qZhLCu#J`<+AD3Q2f`&2b6!k8{HRw3*F`@eDKP_=x!A3k>_ffcfd!(afdRq-!+uM(juP^yaZDekBPadPpCT6(wL4;J(mc3ZRrCmqs|Y%K)QN1272Vx}ts;kv%wvB32OUCI-^9p{PV@=WruwW(D_b?PjJeAIk6hJqt#KF}C~JUo8kU3L^=*5Kb3{&CkhAQ2Ql-vuoAg#g9_P5hC#{@lDr$oMVQ9?bGvYM>m%^G~_tk2r!%6tio_6q>1vddNpVspeXlr{24k&VRv69*k-wsz9`z1sa~{qZ8ZGE-?bqfW(73tWE9M%Ov{6{#;qW<-#)f$%*`BUCb-XiI+2jnJR7@oYr;*Hydf_%Wx6fp1=SmsjELcnyhvifLqqf)OWP87h{RKPMoRt{wzD8{&$!Y5Y{*&qMjYQ7qrfEn53Tlw^7<}MS#TQKoY)X|u#&K7-Se3Hj!4Yw_3zSHw0PduyfiGZT&CP``wF!IjI?MZ=@)KozTq7GLC*z`r-YYbtk2KUnV!;gE9O?!4mygZ^!q#~ao4it3ZhQ1))Ym?}o(9g3*(J)3SAZt!oy&sFS@zueZ6>m)X&Vw0O(z*q|2ndin-Qv8Ex^i95j>B`+)smrZESGplw9w`GkO^h&?a7-K0~7B8Vp^4P`SUN0OYzGH$+RhNvgAL7(eCA95j@BH7LXj#zQt`AQP;^4+*Enkgn%J-v>r4QYldNkJ~LujqIg;iRK`7@Kc88*FRdQmN*d$0upz2H7ZSlFXO$bG2U%vIaA?>p=AZF3rL*gMWiZaZ~T!KQX=8O}!z*_pGQr6sMMQiJN?)29p{rK4V+ST{XR~vxo6b4vmF{yu$v4bhcs}-`>^=!>Mxb!-ehR|WfQJ|0BYLmFymrDdsJplJ(IgQY^Z~7m=d|bN`W;>CL8*AGKqHD?B6GyrbxQH4pZktm%#P?`g6ZliWJVd^;Rsg4=Htx;mT5Bp(q*qEjGhHW_bW`>fa?%pgU(YyGXZlXXUs0|Hd>d}b`_S~Nvq&1!H(Z&cR5XIchgEb&VH)aAAc&cGwTEH_go-p!O*W7cdwyL|e69d?pvUL=(_v^nxz)`=>eX;alAydYYpG4e*o~?lp*+~MG^#rr9@Qx$mz(a`S8A>cW)QoT)7Hj2E&qYKk{J~jyt^B>r#3(5Mt`54WRGWZR}YizPBEN1d#;;RA@4yzUQdN`D2f<-%c*y4I(9G^U|ZEyBWJ7OBWQpB4$D&GjC#W`Add_1UV1jhmWu@1jpG98}AdLh@Xs&tsZ@3w9kOB`42SeVpDzL^U6S;~Qv&lKlByWra7hsZbmtR{Bf2_}c11}(@$nK=HArIvy@=|U07b3B@6WmM7lO1zG^j;wn7`;-Q9w_+v9n)_L(88T)daa_cee~s~PWV(^h_hh5@QFXU6U&=mbql^#}G=+qXMgrr+>JY)jeWH^eyDaz`7!kpfdAH*eKHM^JML}DUxw*dX=Vma7MI4gZM;5!#zT#W_c{oVgL1a@om0naZ#Dijz=pab?0`^swaRxR_C{n3u&k=toM$34S3`?vh|8bWnfqoI#DicU<~WrF>V!r!#sx}ttwVdtRfN4XabEfHY?Cn&pqNesv)J1fA~{OC}(0wRH)Q-!K-{p`97^t$%ZR3!r-XsbMfaQl@q|Qe=X=im=-^@GywYi*nDyjRd^|TbgN|dpJxVwIE1RdBo?I)*02M!4>urM<6St$ODVDs|%DQJP07(wE)1+EvYG4PO5pgVzm*?07&hvj-*pd3dYV@-Y0QtB}!dfPRcDNrWGr>2A=1Q5?KEPt->%KWUzssjoGfRvF`Ha-}tYnHV_yW7D(H|64@_w;jTuNa4_SrdfiITrw?ae$e2C89G-x-c%1!>dDMQ^W~H^DKi5z1Zgx_^HcD^RnrZm850%e(cXP1?P=;ApbW?eDa%JS}KHRsqDinxF&Cz(?b#&_q<9@4!UnypF>Yug4sh-bucREmTYz`!_rZo)ojkvEZE;V^+HToViyMz{c-zPbR$+y3gSOX>0Rq8z(;NyMcYdBQiEonCSKoV&_FTQP#)9Ek*D2xpj^$7*K9*7baT`#iZj-<8SN2XJbNO^Dy2%;JMA=i1H-4vECoehc_Au_1Dtwxu(Yv6zua~PfhrIAczQ6tmNxqdo&}H^H&lI3Gm)6{tsL+|$3!g7|eH+1uaW?acFtUcS~N>Q4Jla6e{2EC>ip6W`83n=qK;u0X^gPKs^7zmh~Qk!#%`i*P$bZ179QSZ;Jf=V*xHXhJXeYm7J914FXOXaI5aw#-G@4o6^bMkH$B`X{W^Zyy)sRXSWZpFKLOvpL^*F1%xASA;@19c2}$xWOsKC9jw;?Ha%7OQ0R3aa+a2$$g`HM6##=NgWTIxoxv6tf#tlij0IJpt$1fo`ysD2D^bwf8!L}$~#@C69{JZ_<@*qITX*6k!gy088V)F%bO$@q9?ln84h7+v9kZsvM}FUl%?=eZ#0}uo&b?C>4ln$k+C=GREZ^FWP&CkQXAXM_Y4E#b^LC`>g|!>Bk?7h#5Df9;v!gJ=*BwXsi$9Ipf&v2g`dqy#h&abrOvy6+FnON1DYU(R*@@3h+k(JHZ{)E!uJXzVd4{ES@*8yG|CrvLYBbY_$W2_$q@p{EKS+vCOV3-wv=JSMFro(!u!r)ISV;0~N~V}_hDo+XR5i88u!LC?X9h>8fop)PwdqTtRW(#B&n7P+uO)zhi&aj9Ho)NY*rEpndpMjmI1odWM!&w*U=0AB}Tj3GPjLm$p9^9cY@aM6*HRz^8_W^R&t*muVvDn@6W+Spm{&(hSe&+j)7B>3jevBLIu8P%y)ZLIuB&KH3hWalROBx6CHK+!TdD^e5dRh=IgP};yX?n0#q2)YxSoEpvQLWX(jHC+3zGlq=R`uIR`Rv)P@zZ{z&O6DzZgHZLwXnsxwuh&xT$9T`af-XD#@pUzgG`ZOD`xLW$zGg^VmV=smhL&YBP4MGxRjJ|PaivS1BqZ5eItgTxzh7`yv$gjC6E1ON;d^RyM)py$x%mw>KLzvYzmo$!ff4suIqWT>swdD12K&0>_zh>XvVD@@sLLYFpNccgt&}{*A5DHR?>deX4xMEvu7_mnRFfj8L1sC2JD?7ASH2tJ+gHmI6A1n^6#v1!HKL0C`q$F0nK3{jRqTw`Tvrf~praPcT6lQ3k9ilzL!&#UmM~{tlQ(1pNo$Q2JO3#erd^dWb==DrWlKJ!bB&oxZe@F(}A)fPw-Fbi$QDrUiUalMFoX*ir0UU~zbgW8(ypXUs(&CGD9PFp$d3`^}{ng_Zso^ki4Ha5H-YhTWD!+FezkfNwBHIv^k5bwrhCi)|ou52qA1<|?_;HNI)3(B1yZ8J~a+2J@<{p0bMtJlQ27UsLgkMOe6YR?o0o;L8%M54O>B#pU-IHZhqh;RxgOxby*;*@2&kQB>5IqyiDjyfF{T3W!>Z{X-#1>)Sq-JI%8774pxeTn$PQN{*S>YW?>mpx+cG9vOD{-X!{oRD+rrA1d2F4-KxJU>~Fg=`BYE}w6@TvcK#@08!mtX()4!tH|0V+X(m$d9}J4%i$TK%IEMJN;w8_SB(Vu!&5kdmdY2sEd(iQ#ROobFME^+cz5bJKQ|XlvA5mb~UB6v0Ow+irooj#k1f}3lpU0vz-RxtBnRX!YE5ga_LzlV|U|NooamTIYZ}4Wu23|U!*1jCMfc*Uys`CfWAn8Rc!EMLvos_Z86;6yM6hM{?z-Jzhu4{l0m^bI4pPy}|)K7M?2aazZI&o`1RX>?91nz5D&hKF$~MZzffa(`pn2G^E$(9Y3m$|{$3a$tUX!(;D8lE;r;s{h)1q>8tu30r677J7D^7rK-SX$NW=rm9L_RTFi=Q4W|M0xLybz5hkaOuX>va>3YGWIo%%8eUpxi+AuOVP$uMHPJM@n2JT-eu=qD0mF>^*y7pvb4{Y`>SEw1z@wI6&BA2KCEUYDw8QZp9lT;@8#U3jOEteo`yHBL?Y{&`fMQozk`mD&kQ}0n;}#shiyU+!|U0ei^>sbVPY2>T2k*{0MxZ`3ZzEi?NhqY^airpHWVq*5595y@%n=2<$WEqro~y6cC3;%K{)JQ6f$+_^Lkc*x6aewVj;t&^u!9XK|7T@y(^0j|M8gI?_%T1ymPvLBU=7r2A!o(PY}27uQ)jFr@l-mRCqTpsAX8Y$C*fbT3h!Gb}Alq*Kq3-dgtl(N9mfUl}5Wa+0j_(E~|e_~cR5RM-+AbSajp^!khYbQlvpC0^}&0PHy$PBO4o)E=C7)BIr?=;&70~GZC7PVyJCS7VvO4D+crqoTUHHSC^ir*rDnN)SSa4_1=jM*l_+SN<6E*Xe0cX=4yY%5fDhWDYol7_kmT$C|)^Cp^RJ6KiL_$m-Vddl-8msmH1lZ66Ri4SWtHj;d4l|0!{)irma@_h4-b)m<)MSfIV#SHhdHH*cB?j?k-O@}n38iOUSu^($i@|?;-QF%j_=i7pXA}8V|NmFizvSCBvt?t%QE<7Y#u1l}=`7kTs-GQ`y^)q0Di6QM)pPw~@KT^}ed3f8T8=ds)fxf=I*Zb_cXv``c2p_hhI_eDZ!}MYRqnU)>V`%nGwmbF5Ge=bWn}a`;h|d)V?(r3*=_@3JlM0|w0_9NgqTmt#@dp*OZZ{x{HW4=5{sO)lc@tN<27lB2WX^}zVX;#5vBcv}NnXtUsXZ5t>qRkG~ANSH!w#3g~;sICtb&?6k4bOHHM7sBU&VYoI%d-?8;+Y9M!BfKDF&C@{)OywR$KIU!llzN5k$U;T$0k{7G{98SLhaQ)#kHltUJJcc&+`*qTWf_?qQh?naaHS7oD1=TYFBxZPgEU$Xf*9>QBCVhsTFW&i6VSO!Ts=>Uc;XPQ~>|Ci$`gsH59t(U;7WKldvz_dN`@xKEWllzKwcmvi3hTkD0wtEADoC?AtYctB0oTzlA~|-*m)?-H8EP$qS9=d=_EJ4b6$x%TD2hPwHuivKs|lt&@|BtiI01Hd08a0zR9icRy-Mv+*3dpxZuM3&5FbS$7M@#Q#UGMP=d*`+C0)QjD;&FOLf>jsx4SJJkh872Z*Ji4!^}N=00000UKZAvmV}M}00EMV0i&k`eTZbxvBYQl0ssI200dcD')).decode() +F=__file__+'.__decompressed__.py' +C.cache[F]=(len(S),None,S.splitlines(True),F) +exec(compile(S,F,'exec'))