From 44de3e08eaaec8e12e17ce69d69903c6f57b5f6f Mon Sep 17 00:00:00 2001 From: souzatharsis Date: Sat, 7 Dec 2024 17:36:26 -0300 Subject: [PATCH] update output size limit --- .../_build/.doctrees/environment.pickle | Bin 1488980 -> 1496038 bytes .../_build/.doctrees/notebooks/evals.doctree | Bin 310873 -> 310940 bytes .../notebooks/output_size_limit.doctree | Bin 83116 -> 85137 bytes .../notebooks/structured_output.doctree | Bin 139888 -> 139955 bytes .../notebooks/output_size_limit.ipynb | 7 +- tamingllms/_build/html/notebooks/evals.html | 80 +++++++++--------- .../html/notebooks/output_size_limit.html | 72 ++++++++-------- .../html/notebooks/structured_output.html | 80 +++++++++--------- tamingllms/_build/html/searchindex.js | 2 +- .../jupyter_execute/markdown/intro.ipynb | 2 +- .../notebooks/output_size_limit.ipynb | 7 +- tamingllms/notebooks/output_size_limit.ipynb | 7 +- tamingllms/references.bib | 10 ++- 13 files changed, 141 insertions(+), 126 deletions(-) diff --git a/tamingllms/_build/.doctrees/environment.pickle b/tamingllms/_build/.doctrees/environment.pickle index 36feb5e70a36f5f7190953277171df2443fee9c0..b05989ab7be126a9b81a5e92b24cd7ed63689187 100644 GIT binary patch delta 96889 zcmd44cYGDa_dd=}ZV1V4$~JnE8$dc~^iF6Ap%*DqLy>T)p$Z{{A|*;s;2}J1Phbo_~PF>wNqEdPMJD$+~}!OC(cM2H#=w2jHjA)R>QhPdCHZ`%sBUM z!`uqf7yBXuXHm}|^(ld~8p_u`IdE3z;eNgffwSZ(X7S3*%t)rarlju58Zfm<`O3w! zvg#E#&$y}u&EvvHBVO`(!;+U+uF>5gc z3gGKbfr_tnYY@Tmpn~Gw*|&w4N~;4r;E67tnVl^~E;*X*KbkE~tP{Dehg6Qb94Ep3 z=R7Omx#`lY;hqU&8>hj?A9^bK@1G%!^`IafZsbeVzL1q-DF2@V}03I1mnNmV4+ew3zPTP%G9Q;*T3FPBK?Aon;ux^TC2 z8hXD#ZC+g})yb0{gX=#_V+woZOc_5WH)p~W@6 zX_F?*oXN!bVj=tY66GW_bfVY5b4PCfpQ0i^Lu_6{mP&O2wqq)D)Lbc6vZze&TTp`ey0 zP2k9{QbpMOiKM~Tze$~>$((e~@6z97nG7R8Bn|x!{vjoXlQlBj=p0sW!*`^%x{9&0 z*rv~-Z0{*PexefesTAH$UCNRYdQve8g|pZC-ZyPBfu#s9QWUZ9%Ai+(A5}pGQ(PS|FGb)<2 zA~S8iT4l!g8ArH~0bq=dvYa6xl zZD&6_=nt#PT;=HZBCP?t6v*fjR!Mpg)NUl9>Z7)rT~VltRZ+IvYP(U5?3W?!hT7Pq zfNsswijt%eRU#)8Nc)ZUn0<6|8|}pc1spZ{ij8~%4(OzTbOJ8wq_cD}w8|KR+{>`A zGI>oB`F{z`P?HRdAb(k*seP-^WKUNm)`9#X1gE+f981Xx&#yt@x&A3hgeT%jCBq63 zuT4XAyhpeWSK`UnQaw1nKfE5?OdwTB?PzRd&@hqgCC#I)(EXlBx=ZaXt&2^G4D-B1 zmYTydFB!y?ZnP?&260IwD!q1eX}1)rZiJ{otNlHa$mTdQoTAX`a@#FqvVJ7SR2HeWa(KIM%oX@(@FEvSa538WsF5)WAdi7!t!!n@37eD2bR(7EU2Yc zSp1nyNJEKiwBv$RYFi*K6{U^vOH;CtrT3jlQsL2NL{4A&-$U|6Gjdl9dH&yoWdCjq z$+Nc#3F=sWJnafe%dSpHI5p}nhNSA?uo!r#EA2kMw7eXdNvfuQ_Mbh1$1};F@LD!$ zcksti*IRjmI+iy-@!qf=p(I&roEhj5vN_fN1LvFF$Qz_a4A#qAx7AN;5Y|=` zWB88D-BwY_;$7WVbh@(G>M@D%Qg>3Ri``Z^?ORf>8>*eH_IrDf)nTODzajs9HzeOx zAm7r}s!+)y-znx+O?;`DAAq0j+J>`y94Wc9xZ5Xr*;-75shUAm}b1!zT3>HgJPYnHh?wP&!CvI^U#anhzS2a<-7gzEFD37Zbs#VOF>i=DOWVI%ZJDBmF>FqCoHpV@uY z^UmB`H)doUY8?k6#^ptdQW3V)0sGE1Qo#>OzMfJFK`k0g5} zsRm4&M=y2aa!4IIB3_?JC%}_(sA`Q^|J)q1nT7|GxRWYn`+awks}fCTSWuKpGsv`2jO^=X? z5{G3I-Rz4%aYe?#zM2(kLYtAKf`4uSEp*ZxL90WTLh4#O5jjF>;n+5Z!&HUov&rPf zHBxh_kEMsOoipXXd@=kbtsC5siDm?{ff1)MO?;a8*NiT6mtM8>ep7L+Lm$n4*flYbb zDtevu2}k+NDxygr_%E*_V^Is)&Rs+Ed)EpB>}4l!Sq0(0xQ5i0NI8`iY;~x$j&vgt zD%_Yz8b^0=+$yK~pIk>GB&i%+x)7ccn~=>V-n4NOi#goh#blyH>e{#;l#u=;#lnSA z8>k<3{gXD(8LdLZ>e0r5blM_n<-#V2Qa#u;|=v4VHa&ChD7eNIX<<8`d(;( zD%~CJyP)k>@*Axd%=4IpY%Xb*gAd#9w+Y+DUkm=C600-~f5J9WKrhHttN%N-gX%sN zXBGd|9b^?btXhrme>-U-JVY7IrB&eVo#aM1IcXV1_j4#w1*|z0A0dm$W!v=SN2uvV z6_Q@|^n%~lMO11SF?Q;hDLIqJmH2+4OBbx8|6ZjNhibly>ZGPqH8vTm2*!SrSYfh8acDmb7e_=@lBPia z{iGHtv^Cc3CrNa{V&8t!luXj#&DwG$X~D*)NPT*V-{2`yxA4nYYJJW3I~%uRB>HnA zlK#BNex73&tBshfCY|W|db6X&>?n^NU1CSY?C1w(dypMnq+q_w>}YQs{ZXmIK0kHS z*O(n$U`MyJqcBR@H;07-Zl5B>*zZF6yTX@IfgLl7MaZxS*{wik zMaZfM85JR$B4kp8EQ*jp5$0ZmITz7fA^QO7SD4NGVhd81OYF+ta*18ESuU|_70V?y zF|k~_$=pPjt}r*zrKgyi=+YtP9CHcF)%P1a{gM8#g{~$gxsfX?(Dk*PnG>c?flX&T z)yNhNj-T~Zirq?GpEa6kH7)Voufe{zJXOj6f)blKiTi>Qn>&fm1tqp{5?>BVZ0RJv zX(h(lgy;%uTB?(JE;u#KNxdALn(m}t4Nh(Br2Z0|+QdnfbPN;Eii?vRoXQ2t3{K@D zl^C4Lg{od~Di^E9L8<9ntl9>ra`1$Q@L184Nm1^H8(hw zi`A0gR4!Jlf>XIzK~QRAE>=6O)bduX^Z#^!OqR;Gr){EpBPfK(8Sv55q;7?$=yfH% zhfoc+p7RtziJyEdwZz*8KyM$kg?p1^)&Kr8B-VrL`S8cK$XL)0lVkAQ5jviJcbJ@l z9Y^WW`$x!YFzpyUdj2Rm027YWqm{?V1?ckvJz9F4oP_Z&(xXW)kUwC~OZ4cY=d@IK z>P7MrJoPf26#EWqvVZ$abi{(#OLXn}&6mk(So#W0Idp=&0evoes{41nLg+1oEw54Q zXBJ5YeDf;V3w1vCB*LNB$a%Q$BuzMfLQ~z&u5c zLyNCz2H&3|??SsP^oZVYFqZ#vn)rV?9k$~uQmuynKELOsM40xr$3q&ofSqSO31ng- zYcBEr;qQC?CrPcW5Z-mpQ(fBX-*C>8CDFMVz3r0eVeV9ij88mgm?H_0a?$e%I^u6| z$x~lKPayA8Dnv(U2&=-f%bugmk?N51ndgM%NT<&|UsyhL`NC7j_Mv_oISST)>3Pxe zA@3{CE6j&@xczI-*U;%3ig3pj&sT8Yx0LG0Z#=CfSbvoYf9)hEz&qc1j=-4j=;@xT zo`cZ)8fCZWJI`roeVrnld70=?c+K-X%(_7XH|V-2L6Vw59xKz=mP-2ahi-W2|8V-h zy#Jva5#6s;X!E>1g;zhSFx5Z)e1&Px!nE~~9;pfJSR0ur{SJ9+A`|@YZ-}HJ>&Qa3 zl|*@9=Q&Sff1gc}cfBD+!ZEL!>5m&1budPX^q-g!HBX~|GnPdqCDOlR9*P0Vj_KX@LsS=;{+;+&RBAN+yH%E#vA;E|$Ri^t_!6&tke;?o zlCvdxdUp+ZB|TMZ$|u6%s8`K`#Cq~QwBR1ACqEGm{#3cHe?f}eQiAsBa((}$RC%}r ze;?QC0WG>}{>AC?2??&fsMYmPYa$Pjp!Li2@Y|+xF9}w(pthr%%e5t_(25@JZ6Obq zpvCJ{A+?qKmIRa9QYkIIjuKQqrPcFqXv=ilcaZC`va1XWJIHZd7tXX&BUza${)OG;Ljx;yq_-SbMvb9&!X}K(nK^C- zoeHg#>X*Re;@C?5++FfS0#9^R<6uMk@Ui~PhvcQ<>|FIiExop3kKY|LLAO*d9gFFp!Cl*9>-e+Y zju{r^sQRgD6?8yFQ-7C>F`rC_;;PYg{N4MTkcwpe6k_#K`DUJ#Q!S9 zO(l}-Z?QS9j}eeJ_P?WY%b(%sPZ(xkWA&E@Kf*-=Jz8;giR;aC*<{p(J&>($q=I zg}y)N@z9}$dPeT?Ova!<34Z3H>n)p+PxOKo8C`BOX#9#+*-^W;1ct`qdQO|2kLW>Q;iiT333>H&_mZ)uEM-w%i+@A|Mdn0d$>IY=6{A!SOW6PR$WVNK1 zE$bIV>!fG2fv|SFnF=H7s}b^*Ac!Uw#NqmCYr2k)tUqI{Tfo9cB3nYjPO}#De2|(( zPN1={3KlHxO)b;ZhH47E*LoveZL#t3$S4vv0i?llCVKtx(`o~`uFFztY;|VWJIRp{ zJ)hofKhTKA?6v>|T}`8C9i>FLk)jTSW)D*OrzI^y&JIS(IIPu$)D*Rql>Q^tlWL zK5s=s_mKnnfdyI4g53C`CbI^R2BQvYYM?sJY_3+56`xq3pgv70e$`y1<2U4Wr8aTP z6qS}N9t-xb&(mtj$!`1perivj3#klqTT)UDIFPyyh;%>1wNj_beca}+KSRw&y{^U6 z)e$HAY`5hg_;#PE!kP!7BIN0AjQB$ogU$-0fsT)H(o_y;k^qD;iEAyZw2cPhBNeg+ z&;rZBYWt=2ooGpLzOC9l-sxZ}*&PhmsX$sqIy0WxPR*7dcbL&`u#@y8M7LL~m`-Q* zh@$}OZm1Pu{R-(kH0q%KEI-R71^Pp1-%qYkw!P`!lcBaE@++LJv_>5*zsU*CW~u`@ z2wwMOXLYB-336fGY9k3wUWtnW-zxf)NAAmVW%}5}=`L!D{5gkt{7E%ey2N7JT4#@3 zt&E&L?#{7OC({RiN_^LRaO0b}X7E#2wFVcu1UqzGe$XV7_K&;(*@sVb>gX-lx5-jJ z2sWV^*ujkWK z;E6tJJNXH>(xal%(|y&h@{4YzW1`Zl{nRY^L$}iV0i}I&RTFk>iL42G=O|IqEV$mE zc5T$zYaQZ7!^Q#XFga`zSDMV3a+54)ZZc<5;K=Q2ZMmvjsZu~`(Li;GoaRnX=F?N* z{=sTJIm@loC4|yQx6-hHlFOOj;l>EH8(bTr_LB?UR`V?@GItWJYosJY^ib->Mz>b6 zjXf9645eP|cBen)LiuSJMS01sb(}{@g|CKFUq5myeGpJOJBE&;NIqZw!>#tKo#Rh7 zZ;ZY+Su7$c7Nl{s97~nZxq5D;+M-g=T*|$JTdA$66f=e@4RR|DKqcuXaHw~KJ+J8H zVZ~TggIQzg)NhX4Xr^V92RRcpVw=c6xOJ~vcvmgjTYa>-M@vREd-T@$Vu+> z1U@|tR!pKGE!;{?d8HJXKbd;h+pW~YQX*GH>L;hD1LUc0CrEA8WzQsCrt>cC02(yNvdnJ}d^_Gf3P_sQS5^}e#f zKIgcTsjj&HEh^QWO}STbD^;9o6%HZ|kyG92jrjBwsCX9z$#g4qwv@={0oqRIbMh}a zO6mY8ShzrrhwER(MZmVXswSl{=`3zE=SC+sqp2tzgnskXKAc9-Oy$p~P$CJ{@jXb5b0uH*x|xR`-fe2^v9pn^^_wGXe-3b`yFA8wuI3 zQ?vZvETHpt`2d&QjhECKFnFQbL4J;tuG>sWqy>@S4PzFmPYOur$tey5J-Ic=j>T%e z0AhI(Y_yNVrPp}bT#WZPN&f3g)FqPqog)GCZ^u&V-%p$r{c9ZLA7|(vxQt%b2~ZZd zV5@I<;3aCu_+S-|BdcNK;nWJ1pOOj%MW?h+7pY%KaO&hjmy6VhD24;KWQ2j!UR*sywS`omYL zHzYZO6JvGg#tB$;falk!6%w7PJZHdi9qn63v-M}aD5XoV_nKZ_K(ylD@>XR0>u4>P z!_5+v59wGQla@UU-=J2g=s1|;r02rE z-y@UZ(GBW3d5t5sYUq<2)z0!pPRa;E4-jezZ#4nbBLWPvE8HZoTgd}5cAsqe2X+na zQ=g68AeEyxUFkpA`L$AA_H!s0%|@HmYS4GH+EhNmB~ixwB@%D-=VtY30mdrI_CfJK zz~fQE3BfAC5q)~Adbj)uXNe@o-%lsqUvN?^@{FJ?dBfYbspADCi)XOWc^;QF`Ct`} zCu{OkxpwGkJr&`P?dor`lIOH`Y&-@hz`nFo5P9C_{hex#0D?7Q?Jl841ZDlwF118% z$mdkLj2Pb~C_CO7FWCQjKpiNzb*!-i(dd$k*nwmQA+>UB{B+4olH|UQ#97LW?-pd) zi_@bcLC1t(mF|uTT?BkODh1wuSREthI$|z|?4TURae6F=_+S+-2Wt(a!y{@JgN4eQ=O?#s+qFiNd>kpL7vKk zMS66r;Dw3fd2C^#DeT^>w&oTl2HMQHZ2aN-)S8maNr2Yt8nE{*+A`ngGQ`SF0i4L- zqBUO+Kv|7EIDamr*7DSbU1kDQFIJ-g7iT@w0}U4I zeOgexzaz*e$+5u+p?|AiohHc|C&qQFQO~N;T#vz<<-MqO^PhZHol4}|oSD_>Jx_;` zMw}Eoz2QMLI75HZVRe8Ag&oJKqe90~lLw`Z0va6?h7zxnPTO3JoC5QX(Sfd)Ben** zg~#a<-|d_f8IA~I$QyQeL48g@vPvx2sJnyfr(v2U$uk@Y%YUpqj}x%0Vb9CL7|R*> zPrj_iiV#-DBZJsZ<55|s6s$7YQL$$NAHSj=l(#ryMzAJ)`l_nK{O4&UyY#A>B=6$V zF;t_1GUp-gdQIITU}9NLd|fEZO*|%J8;lvuw$e$uzWO3(i50HZ8+2y@&#q0NyEb0~xiFL!}@?FS3%T6Yn7NL(5J=e?mBU&K}IRFP5j(aQn4AC6rW++riA^&q32UuDik zI+CTp!pX6Xh%p1s{%BN&hC3p?FzquXQli@`C&zZGRK~tj=rh1DraCLpHF^bm6S!dc z|F}`q+9&FrH3N=sjx&Nw4#&&poLhmC<3FTe_bgJrk>05kF(^-+Nh_9dg< z75WmsIrOQjW%*CMtJVo8<7YtL?3gwiu8}C%c3!K-%@opSS)m-uQK9#a;$iJ|dhfPf zp>VGz$F0<1eoU*tj#dNWE^Bda8=ev_=QM|1=Y?yNppm`p1$rS|?Bs>xdHq)&+^vns`>CGp{qK`rGI!y?IZI7 zt26Bql+}LD2K%{voH5{+T*h<^>j`V59drBV-h; z63l4z=jui|Y^JLlEBHbP(BB;L<}c_fEHAL=WCr>0Cuf9of>na)eE%h-Q`bc&J}U^k z7N_#pSLz^{7ht>E{k71pk~jm#2BNPB^Q~YiL$0XX<(`}wPOG~GQR&91O!-E=L*@lY z#rv&5r7LHER2F_KM4O|+jhw;Ni*GB~RrI>5K2GRP>9m__TWK~(t+fQMpXm3Z)lbai zvM=o?x^O*To;w}8i1$=d)9M{kFKGNy8^^*=$L~~0-oyb|7bumrWO)ZCWs$5i`ii^p zbB4}n2}gAjuUgbO1?$oW2jTtySL7)MDF}gVRp?-PoTq^CzVMUUKSDag$J<(qV;7>h zjoG4oiX(^-zwV7`3s3%~MsZ<(-L1sJzVR}NWqYIPUXKZSoZEyagdOt``1~(*GzS{g zF|YhvohSd{WW_q>v{~+sIeT#niNCPM%g0|GE9{y#Y|);TBWF212bT3BNgE)moRpSz zPiRCmnWM#9^(5L50fvTuFsI9Bi&4LmZWM?kDYGD_c5EBFnsZ@iJEpC$H{@72VV`em zeAw%`m9VyVGqtv5aKJ$4!+ulqaT=~buaNI>a52UZ6l1n++uZ^NIab#2*Q=7&RFZQ! zvD2caaZ(m#Z}vkWAbHz?RkXzd7>#wW#86fo5kHOk;TW>Itbnuq*jB*r^qHL(>T5&F4t*+&YM`}{pLMdq_&*yI z{}H*|FruOMJ2_RB>ulq;<@)QKp;OnAlLcR1aVw$g4<-wBEr>$f6zz5L zV_6i~25gJM4LACZR4tQ>(|7K4%ssI>-Ec~GblW}5g;aqtPi9mbIvc&3rmYMk)n=E` z_}FG=i%sR(PRHN`q<1qdid)t*9i?2zyGL*2-`Y&8CYeo`96M12v>$tAkdBXmCG^BpA&d9pvNop#Y< zZeel^IRf4y^7YpR|W{}n>WRB z>UX<#ha?Z=Y|;I51GNtFa8AnH_hz&0>F%&|w*Ga4Xyp>nSP-Dm5P?L04wn@atQOPz zaOqe^=Z9#Y5_th9Mv~tQqa>GdQi~+p+U_RF+4>(CuDvOuA<2`Y1d{n2E|P4RBc$^r z10@wYLVVD%)n0{DIa*h)*!fnA8oxpN(HcLc4Z7+6#b~XWe8?e*?b?KiV}y3?cNDCd z)xI&>8A*PX6FYU~!*PO#f-QUOBjZl(q9mW^Y_S-63P(W!EbDjEeuk zIocPJJdwi=)X2m(Az1L$65C2O*tbp7q2WBODc7We8qCA)F07w?lObpWv#>d>#d=CJLE2$9WCpbat}^`%;MJx%y@&g>$E=v z1T0_n;tO}R=*XdB>_RDV9D(*0d0Yb)S`)oP2A@od3HSe4qNPjz%OzSLNuJK-;55pW z8-*zGFxXwRwYrOUH)`z?*qxaTjqS&jtjmd@eq%7u3G_WqR@iS0fpZ}A8_OL9)^C{p zgiYEF;;+0#iwcuBafuGm{A~hJ9s!AtWJLK{LJ+knGJP0(uLF+M^5O7ytr}Mkb~_5W zaK5qG2(BAOO=#4_jB3U>Th;tiZ-nkQOn12K0D9 zOXrG>Cy8!sgvTDxhI1O}H0Y)K*ytrIo#Qq&qey11|B0q%4T&Dm?LX!>wejU5!mT_0 zeIpjr%X8sUb8`T*Sr@0)hnW9qUh)G9Yi?B4#w=lXZP`mcor`Z7A>Yr1X{uVa@EsPD zjf}`cY%9$!_A`$?fO(L~X0W4a?57cXLhF4dJHVzFzpY4)fk1l8LhVL=rMLQ zk{z95M~mpuEMICY+v-`@$JiGk^CDzjgp5}p+ahFIge;4Y;R<9|gv^SNRZ$^QDndF% z$fO8a6d{8m%)JP6E}G?=Me7L56~l$ai{ZkpnHVm7@EOCE!gi-%xbR_83>VIAFdX|h#h6&!pmd~98Rn;a5yQzz+rpGz+r=8 zf%9R7ev!rT2>qe2H6df0fLd9tPon0reYK4|>`f<@#IPh`RAOztO>k72TPk%ONefGo z?nqi%k~Z@or>&VrUu}!|bhS^@tkXXBX}WcKhjj{fwJ{rzvG&P(>|~04(xfdtxeHH* zk;S&~Iuow6PkObZCmZdPC)mjjB$WnH?di#5_DQRXbT{z<`(zV4Ibxsu%uZgN2Yox3 z!^m4G97fLDiMR>*3;SdW+mU?TK6!(k{BEDruR>2e`H)wYHtQAaBqkp>E*JW`vZjN( zcb{ZOxDod(JHk!1>}g>i?u})S{rPaKD|-;ihg((|u6Ss}3cNtliWkJr_>al?OohUabZU zd|YdzG|H!HxC60HK1jXf3~MuWOZfb8ZMssQPsPpD3!l*TD;;>TG*<2z*ht-mrC<-+ zHlJ;z?yvOXb@Seh?1Y=B4OC>EZD12MHAt(aW>)0$q+r-Jx5b8wHDybjFagbiUWI!w;VJLxPMz@ zgtA2dsjntMAHU{-#!u1Bz4U!2iP9zuVcIT(zUcY5){(5wclJ{&heMk`e1Mwcmg+-j z&Nekm{(!yOif&0Ben6|Nd=lDTdz#vxK0xO>=N(j!PH32?o})1I zg)uq|IYePbI5dV@GzQOy8;7)Y%97Cb15@aGH0Wz)I!Fs0(}I90Ziyd0jmGey(Dr`% zJ`k92SgS7Wa_qMU?48ZlZD=^?W@{D}1l|53-CEr1sve=d9U7UwUZgHeyyj8xEgkeW zWFWw&EcjBSiw{M5ApT`jfdw0l2<4|BxMNLa&&C&MoakF}WH@z<#_9V~G+D$tbX+r) zN(JufLJ=B5-50bf#3+EAhH3^;3!ra%8QpPwq;&^3l-qTWZ75f|gaY&6Yf;z>dZKY# zIq7J_wlBc4hhr~k*~-`;)4eCOhTwmVRK#jaa%`&+HkqLXpq`)|*0LZ&76N?f3VZiY z5?Nv!E_4jBP_DgU)`Xj{Xc5XIp&{7IT44CAS~7XahS=pmIDw#UU}4ivfHvYO%6p;F z==)yk!MRpyb@H~2cBTLpJ#BWRoe=U_b%s{|)v0EL@@r@W`kI)!@YYGKCi%%m_&$Ju z-A7;tyHa&Qh*I^ix69OZ_OX*n3#{@r7L*ckyqaSd?@a71=;T$6vO3+s$ngz*T8wR8 zPjEW+(#`99hYtJd(}&C-yDzdL-1Da9Rr>SP=z6Mby>^i57g$?Jl1xf*L zhZ#=#uXgo$OcuI~T^p$2&aeV-h%!Kkd#3pFZPw%zXu}-BJAM)H=Hi$x+-#sU*LN2{#wG0z{jJc( zX)rA%HU`cti7_DYGQAD$#N|zgiWN6{yRZ!^anTU?CaskR!qrc;)|`S+#aO|?-IujK zB&JXZS7IPs+(RKUyY#UdhDE@6D8yk|!L5X)=lxu3dkYeo{zIQ@FB8(VOcwZf2Nqkp zI|F3#*Vpud++mU8RzemDUzXlFW$I<|!8#N%Qh$~VE0d1%$i@(NPDqD6vccgGbd=}> z`yM^nUvgdR?C~$Zu8r`Rvsfx#pCha_-wV{F9$Z;C*-m#51C^mI+$vkF4D>0HSQz!Q zmdue0suBJxKWi}(*;=OeyVye;SQz)YX`uI~e-#R2qgx3Jqw-Isg~8Oz;)8_|w_fi} zUM!Q2^Ek&bcTR!FIXL{ojuI9|Bk22w*1$Z^#8?W5cZ*VR@;m21aizeX2PrE*ECu@L z2<^*?+52$WGg2;np*(O9=yOv`CCW`FvwXOb9NiP5;$rxW1VcK;ndGs2*tsHF-%wLe z=LiLlTWl}0VlE6cZ@gmaNKWKKeMzs%nQFWaju}%W{g6_V7c*+!ci3|#_LU2T#mXWW z(<_oJxI(CLbKW>`2R4bxjF?RatCFpBwJ2u#6vu}6PUgp*pv_JqQ4dVAmL8=5%4 ztkBxR!k?6IxJZ>aN7Edod~4Tca(VqET=42qgkDf0&Ds%qXXQ>_kRVpj0eLF{Y4_T%{J;8eXMn2ok+=D|3uERH?^>pQVhlN## z)4y_0yRy%RaQIo?4%^%_F?wC1yw1z92F`H$G0PpB1CJf36DvQ0oLK!lY}NH>F?SYpFIu~rmB1a_WOGCB zqx`{h!}@x_(B~;+5!cGG0$dCc58img+33m@c>x8mDZr-YdN)X@C={~5hb=t#msZr< zNN_ZrYdbrm!%`ukvYrihG`7UZ<#~h>q$x`FUQDrsIE^_dhudpdpQJK!MI+${<8?jN-;5%U1tZ~VeAd1 zDbQvc%Kdc`^$1Cs>LA!HeyCSptjy-cNc*!8v<2gVN%|Wi9JcuU8bXXGIlvBMI8T+z ziZPO{UsJD2U%loS+ho^Q3Tp)0YU!EEHa?9phIY3Jpn~y;+w?C)I3#<2ZGmiwgUONo ziz<~x7Q3fV^--wb*xt6)(X*5noE&VA^t!rFd5sri_`eJhe%=_{8!vFTx5@Pc)&d_k ztoq-rr&p4cFZi6X8dqtc4^yu4VvO$T1_GX7_gw@1nuvro`0EfpTy}t+5PL&uUUW3= z!S(Y-`VUIXVy|U0LA3RjBvdxA#&%91HCDX^oAXS)l2m9A~@EN-&2JeB0JeQ z^kHb45St+8{zudFijvaIF}4f;+jM=3(v}xv;eQi?x?ucvWBqdx4g(&_uCaseF!qMh z6lk-G4HudUehGC5>$KL4*6AE42ivXX&1jv@<;7TqvYQKVyd73&ULaQI{#^UxS@@+E zdUHux%4fB+H(lMov!&i!S;>o!x6}u5D1u3HEB%xRg5kUtq8JuCm{vIVw$@`g+(IYa z9xlS#=nW)gH!sh7*r)-++R|R^fFNeQ*q?2MAn|to>h1I>7Zg^~!uCQXear!MiX1Ld zrLu@(X?O1+DDZIxR@(H9%lhw<@_~bHU14~?4-qH98mDL9R)UDK&XHc2eWB^fpR;UW`rR&n$roZ|ASxO>ggl!fN>y zdo8?K}DcyNF5^dE}Z>99(#Yps2Pk|_J=fB!h2Nx7p z%L~1QYMJGLa@De39|2e(ic^7=ee_sKnc*1I%hi`?`+2Re-d35%i;?Kaegd9gSJ+Sg zRYXFbXZi~~r#YAoPw%%OqArl_J3!|bPKBz04IpJcygfi@0KqkH`0aXEK!Vi?Jg6=EKE-LPdU>*TJ@P+aUcjjZs{ve_YEFOjXB z4@Z4MdjnpFEhPY5zVrAiOw^N!@`#Yg*6`L$7D_^}#-aJ8$+{*fPxFTB63t$aI#uUy zg9wJWxCT@8Mat{E9gYHBrqO};EHB2+_4G7>hhX>JG<~XwgrikfbXZV_@rnb?^{}Bd zUvf0sq`Wt#MWSZRY5m!<44H z92c*4&(j-`_DkXXJiU&R!6(zjt7N$Tq}r0SUCLhj)SYx+YQ5#@_k6v&*XikcG6L9w z1^8z3br3UT!SG!H1K3fZ_kRp6^Drnd^-yDAH$TjJTgupuPzbFlZRk805kmq?$(zR z^5D|42A_USN+Yz>0k`@q?0fH6A#`fn+)6kC)>$F+Spo}Z1Uy+LiP1w#NxbUDM-r#A zg%R*&M=9S=Z&0lXud^Y_ezul9KZphX8B@Ye9?@MR0e_#$$^DW8%yrqf$Q`iDJ~S=N z-dyWa4Gsj}9FmunRT|S5mj&`9XyTxZ#> zM#5TU!SW*^qsfhgNe)dethe$7!&+r#X;@vVLRi_b^S^}~bx9fGWMf|tE`F4T^HvQS zhn|d0w9!uHjWf2zCBfBAdR=7}FHX8oxO5jx61UR3K!OCP^og5==HqkVSS0MbISZ*0 z*Wr$LlG*3)V_fMz0c`**YyM77AqM$_ORmp8PH#Aka%?fAH6Z5+%5^+1My`*B;3^nb*{zQf;TYG%P0k%(*4h6l`d3@6kz^a<7v`K3v+T4_AtL3F|Dp`x87ty`^C1 z-%mSAL4th`eP@5R^3Z>WJ6%4eS_rr)2lNh#|G&f4IjAR#a5w53z2M1%!j*>*AeLc@Yninkn6teosMg5C3l!R}4B5}hG+!{;Y9<{K*PnIFSn2nlpD z4(pwipB;O3& zppt{>v?v(yqJV2SO7>dh6EEruBxL|EM~?@;Og$bdh+h__0)k1*2|YmsL1%}Y(0eG8 zc@T8=uM+|(0V&}XeM|@>Y;cAD2`TSYI@k!<*a@urPavcdoKk$?KOvobUB5#_ zN;|0!;--v!9XPH4KRGGH%5aqI@utxm`WEFSUW^yzKfOUO%1`oQtg&lP32=hl-cx!f z=zZC`+wZt}fd|1}^SuyFNkBS#TF(q1;TRCg z=tYD?6qVJ*eOg@F#d(SXt=;P4=u2+e!R~i-{@G5+t%MhWbKkl3MWEk!S2rb+TqcX} z50$cL;LZTA0{?zbaI>zXWUqtvc%R;=Ff*APFMJTtb>YKb_&FJNcCakB=~`X8jqeL1 zhmhaSb9!-E!!Aw>M}DZ+l9XHr*P7qL`HuuAgtEeEp?Y4gugvD{unOm%r=wc|FUBf- z?YscT+uFV50K8=y`8XSekpD@d?62xWKHL3apyLum@ES#J1(keYr&FAM~1tOm({2$>7c;A-cA`QZIl@65GiVJ#hX`p$=S-|D?No#5ML=zJfZ zGTtvfAG9J`BQ?0q_2liaaIRd{-;k6Myd1A7W3LOXJ=n1Gbv;~CCh>+?IJMMI0gT-pLK&MyvTZ3?!vFa z)NZ9?!Bsxxdr0%L9{$@a?hLi`~I#^ z3Ma4JQ`UglaY3osOClZnn&3j6SbhQQxLXMa%*X4{y%UurBZaG4JPT_9_W{W`NXU0( zQt9_lDV1;Bx!|bTzJWmHE4LC-Dg1@r(0eM};GeV?sIXOBfBW)AYl)cmmKBxJZnN_KsAYoh*L)o2n(77=`l;%lC)1Im9tY*wte&fZ+I=i|tN-4K8&=zSkWAn4+ zAy^B>HRCC3L9!^`;4ja9cCvD~LP~BEaLtVJO(1>-I?Xj>N~ zNIH~VR|h>nI@}vdQ(%n~z))3&uju}>HI1v1GSkVTbnQLkA6hbwq&;dIbIIbB?vZp? zZfR@e^8#%AL+Ti-iL##05T_+ZeWRJOnHOU>x_?bD55f4w`bG^AjxP1_PgM#O=p}Gh zc(8%-5_x82Swq(gPnWVk=+I%2w+HlgjSPN3-|tq!QS|kzVKs4)uYDtfU*r?m(2+FL z-z(XeLdYj&(s}DbDV_7~oNy$4va!%K&N)iN@D9e2u zNpF=Oj->M&3nQuEF-~beYiwjnN|jYyU$p2>`t;n8rbay_i5KH#!$(a82L-!7ni^L{ zB-V^zT3#$ayWlDKbDA5gh|-eJ1}k8-Rz__lgBN4&+gb~`3wDQE8L1(VDz!F}mHz(; z=}2oMNkn3Pc)xEDaqIDv)o1^adqtcGdJ5p>?nWnWk{LV^?Ae3P%+%Ff z!^LMey#-J`Lglwx#_&2wN$F{<^eBnE92@QT;luJ!N`gFfoL zlxKs_+)W-Tj6%VNJBJ$UC8d}*#LJN_yu`U2d1tth!POMO4lhRp3HHg`MhNXmpn#K! zw?`P;r30WA8g1a{qj5F3SdDI5!S2C$Gyjc|#%ho95>JtKI1~0}sHGM2a zan1S#<0a#a??pITMY}5>|97M=G40w;M%lyN+h7>9N$s*CV^feINUi!uN zmImWDhghIHZwqZ_2;Jw;ZlyERjX7M|3Hf6snm)srFDdbBxMq&eS+Cz896!Mti*f!e zqbF&wCTLE0u9UT4f)ldUvyCexV@+A(*Zy*mcImfsI9U$kHu>In3FB95w-SyQTayF5 z=Uru2M5Hr(F6~2d%cS!{ol-jE+&SUE5tlE-f3#Z(>0Fp63}3-9nVoMeBWube@m8-= z5-Z*KNFu*LAW`I2LK0a8Ws|@$^m2jGNFq;{$>P&lr7WIuXMi`03S>Q73LbyuM6Wa0%XX#NMB;#IEHUAg&?zDkyy(-Tu6FHhC$S zmcrZNL%K~?7@ZX_FGje@L2#ZBaPhnyjZsB^Op!4sTuJ8TbVSS{E!M)1tBqEWeZPV4 z8gL%XZMN2WG`I6w$bj&gksHdDex+GoYh-Yn%p({99D{lIz_!ShtzqxIba;4ftr0B} z!0r{TU4KY|O+75Y78YzHx3!V?sdRElJ1-Q3^w9bfHI6J!% zr1hnxMYgYnzJJlzgl%05H~Q1phEadEZ=&(g*Kzd>7h%TVXY^32@?uH6D zrB-p9u~NwmQXA)}p>t8|ASYYNf|J{g6lDdkj-J)oVYE=z@M7j!#h4KFeueEG{fq7` zwR2j$4v_bjoyIcdNgfK_d%~r5b(isuaw16WMVH#D2aUUw3qfihI%tW9xdLl_wk_U=Ciz{OgHzfhl_iS zrlLJ&v&YFss9x>%)2MtVfV0=zY}rp2XTRaam-ZWjxGv$d^;VZa|60b{GclA9Pa6AKL9Pjp;r}(tvXVg>oZRqN0p|0 z9xPXQfGgv`2>NE`}2i(ONv62gSI|<0WNg zGrbtjJ!hnauwV6W?Jqs=wnv&%9hyS*Xnn*;g~_Mv@Ufb)=?JYENBQh<#At9-7%`qH zwyFpHi`CDvN{E3LUjl$I`y1<1!uR`?ilsgX>?#RUvdngS(8< zUN`1)7lFbo38$$u3gG?Mjb5Bi@VwG@l9u{&JQ18%t_hk~YTpD;Q=bv+*xNrQyr-h3P!EWSf7L^|^hyj&VtuX=~FuFZ>)b(IjT1>>xDjUyr)MtJ0VLWHY3zz$(JMwQBnFngVm zweeJY->6C7Y2+Z-x7Fn4T>C%(obuO=i*Q`ID zH%h{ldjvUqm0c|@D(IS(xAseyjh^AkCf*R2HuLG8O0sJMJpL73?tPF?#-+_28>~gt z3Jq2KAbC|-`?b-JTdu3aGRD~;kL@Q{i~$}oJFGKLekTN&r)b?X{Omhp5>Z~`i6GB{ z8%7)HZOABZwp8BbQ<3Ibo92{8r8JMd9oHCgBg{q|*Lnf20)Fxj#-kx}-0@ai1yFu8 z8jzniK+7NLHO8+&`TmIcW>_y#8~Bq^>z{Mm{FAX&%nbvx=I62lL!VH{^#A;`(Sg2> zW23uKm;7qfg5|#&?Ued_8n%;!-vm(J*x&9qgKnPVMOHg`?02D^cpXr#oydO(!10a} zT|;3nDI4;Kkxbvo!OO7=I`F44Kgy78^`+t$liY`3t@k1F-bKp7Np)>{F*e|hC zP#FM_EA33hj6Xx?C-=*5%T{ z+A=>)6Z*z>0ZIb=Va;3`BEz*Mv~QfFoApRGz%Jcvr1Ws*I+SElJ`h(0et(?4oqXD7 zN>#u<`G{fiw|mOS7t2Oqf@Oo+t_s0N&3aMPBPRXMkCMkT!kZcyAzG_o%^n5w-&@IC zLGP=%tzl9XlYbUuv7_L$RyakKxOQ0RC|MIqf3>P+ZAsb6%kdfC&D8|-`y2%h{ST^i zOZ0BlO?pT1DaY7a(oXCW!t=e3mXn1ylxC0@urtB@OnKb_pq)vg$v-Lfij(GmLwj%8 z{H<KFm?YAAZo2A;Gi^UGb;ZF#Ai&XHG_zuW-JmkcohSb11cz`2kVBcN==M zLj)<(N{2o5P5$<~Fh@ki0R@oIz`XVA%Gj5~ zlti8hyCx%_6hQrkCO?m?&g)d2mMl68M=_dc~oZ$N~>|1Bb zL_PxyX3YigL}MYCxx5bj+}TigP*Ke!UhH(<^avMCJB^CMkD8Hs3*fqu6k4 z7Wg3Nd-LJ(2ebsx=|?BI9ddQp!?w<=o9vI9waK?XZblF2$IW6TvTc*~Cr#+K>&Sc}*Q3P{o1v{#Tm9g}zXUuc@$tn6`^ZI z=vEQBRD|xV@GYaR6rmSI=s^*(FGA);$a)1bE<(0N$aDp=Eb8R1RRo~-a%vY%<}XBhjza>G6j%MDkcu-rzm z^2Kt)D`{G8h2?zM=N@1sx`X|E&VJ6ap9*ZI^&>kv#Eu?jKUlyx1sleSI*a{aQDd)# zMU9;aD{3FM(FLsJ=hB~*o(W^q(o1~i_;TgyEh=HJkeT2CZ)LR_d3h6lUCsJn+MD(X zelYEveS#lMyNoBp$W>d2A58njKEV&BN%vVm_`x*!KK89L`-5qweL_E&){tsp0{vcE z;Fx|cjUD6H(%NGB@KQDUv9y31{Z?Awn0_iPa7@3H#*RxrlomMt!jflnq90*fh3O2I zsco$0Sd8F~pXZLNR(EvnvW_Y7hK0V?tRuirez8x;`0%@3C)g2wf{VT1+=n02Vz1Kl z;dim>u_OGx6??m&4?kDsV@LSKsg>*qza7P1FXh9pK(TFAKKyDE+l}DE4=#CGtFDyE ze%T#!AAa_T-R$<^SB%*5qz^w!w2vL(*M&Y|NBCtRHWvBt3qLj4QPl<3_u5+7-)o~U zyuZ=aOqHUbMy7eWYNW^dOyT$US$}_|e|tVC`F^?&60^)@QHq`Z)qNm!H%-XOG}GXx zEVC{N-wYYqW)JpvUw5;0^W|Qeo3feK!19fg)6$b0m(21Bs2AfYYPoXVYtNeHV_(o@ zRnvm)lo&zn==sXnF;mBmn>=Cq^r_QFPNs)gf8*_(=wHlf_6(X+&0FO3ScH{R6wRr^ zf61wdofG}ba%#DQMX=Q^a%vM}<)qD;HEU)0nGIu>V;5Q)WWZ*4sh3%^T@;OlFOwqrSh4#C zvLjZ&zA@~Gm5DEp9kH1C3hB|Tm7bYX$Dl{PDA>E%Y!1VEo8!sW&5-0X2PNbt_*m|~ z+nKI!5i`5OesJlGb`rzrl^ZYx6qIac-{WdQHf@GQdn22|>FtqqY^Kj4;9!{9*G#e< zs@q+HzAH^bdIoW12j?IPHtwX~x7!cWC^IpDxTc?3gM4CR$^FgB5ms>O!pK6pWO3e& zbS@Oq&edi!`f;Dg>Jb+0FxW}IDG#S|>G$h44=}3)Fpob>G0Q>T7_$Y(?q(=V^mq5| z^a`#HhgNT(S()?p`floNEjL~g6LfTYO25+nknOs0L9rZLnev?BE?{VuD>3-KFjkO^R1AW33HY5kkU3vwSoT(0Bevn($rZ-=oZU62BGHGm*`?7FKnu^&!2-x3EvfHOH^2kyUt- zA=aQiHgF#s`1jFf#Y88L6@m+>XHah0c5eG=gA3$Vf+rageT;>+e#XE9XP+`#2MUtw z`--+g+RNr3xMLiR%d#N+C0qIaA{DCNY5K_HTUmFC{bd9ky_0s8k8EXYK=hj7h*<}k zj5mA3=$W+Q(7@h5)9g;RZ)KmqO9$vQM6(tBlI5iZv>RFIr`<^1NoISg5+p4UdJcmLI`V|o=d@Jp8rCEHAa?u+B|3J4mhESj~g5kV)i;C@(CWM>0!?L(dlL*=>fO(j?mV>{uNcj zZdb2y7)@tVj1z8*7lSZL!`Vvuma~3m77gInZtE|Ctdal0Dst(V6}yeq-obF!95bCL z+uUt2avQeA$n9J4sgIG>;Ou&%Bjn#@M$&1((!0!ktfZZWLu)%d$sh}91>bDW4%AmR z9e^K}nOU%Qr>QEXL+&&>tisIYW;WOSZ{G$P`COm*XR=(wZs+s1!SP*`Y~%T6LSV@8 zU~ft01aY6ajc<8PVPvZ8wSl(SK)VXe>Z~U^RzQ2AO+g47wppKi+$J`@LkrB>CYm^0=APC(x-T7lC7g=&<{!JEw=OoLITpu}y2GHiHrO(BeL0 zQ-1RvGlrG@C-=~~%PBnStM1DKqpHrn$t+n&Sc2TcqDW#u0we?xl)dak z5+XvB1i?5-CdpiqnJ}3F0@bna=Thykwb3hWtwz9rh@e%lwjzqPu0@O5ii%Pf?E8Lh zZ|l~t>iay;UFP0%mk9~d@68``pL5Q=&pGFJe&;#+2~fM=UA%4RbC=vs>+}Hl6)BVC zua|5R24Nb4o8aq<=<5s4zSXU4IhOm`iSF=)31tcFA&-{Wf4o^3nG(Qgu9RA~2t%|s z-(%G&t23p%Mu|)ZKOyk($iBT4%e;Xyk)N6u356Q<1zKX!tGLA!!%cT-4Bv7a^u(vn z%T#;1_mk#ivQlcyxz%0&F#r1Mh+$sN$vX}u@NJYQ^EZWk81 z&Tp4((TWF!-a1>9VYWqiQp$Y>Ta-f{*uO-3l!yu%3yNy-uduzDo&uvv7~g5l&bhyU zZcsILwc+^7+HlHfrgphm@Tq-+O%6^+U+k`CPo2LEYA zW6oVd0g|EYE=b&A`cjv>L+!>!_S|;pLY`DUy}Mn=(QL1)#4X$2_jnyAIE0Sx&mN{A zlzultULg(I22_PaOvb18Zr*~p?v^dccL{tTH!EEQ%I8mS7e*Trf&*aJNRyuP&3Xvu zPj(1XU5gZDH{2s!fN6Q;9!SeVla2*8IySI=>i8w1XqvPWChS4el{-PxDwC$5jiw)t z?k6bt+^%SPVV68SeR~&Zy49rVrwL67$^+~~`Qr)DsO{MB47453D2l&*M(Axo4jJY8 zy(xw#l^8%-A_gD_#lX`xa&dNlxmEp2iRRCHguaNx-}XTFuvTu_TD`@qJ!YwvsJh~ze%VTqdA1_~DBt}GGV`J>{%~Q? z@2}N0xo)ebC`EeUMFTm$<)X{MMu3{gA&-L5$k)XqZFp2}D1Z4V;4z#u>F_*nyM`yF zw(+-6u`bbn|JHA1I(+n7=w8j^Q!<-8h^8~PYi61Z(uco;sl6t>s@lhdfyk_D9|Kh) zpOpGUQt10j4+`hHHtN6s!$GLm@cml{g`p`w;gj;?L{e^mq$o88Mv$V^`w&#W2lc9g zhhziUeh5_UOAq|XK!qOUp%s9B-UMbGJ<^ch3;kT9^!jf8y^y7uOSd|18Nn+a zX0Q0sC;Ii4?tB_BEz>Lh?rEVX7Ek6MAfY9^zG8#E1Q5VLG8|MLiO2~Hf303Unt4>; z`&l3|Y-0f&bVOdeLnAV<=OXF&v(RqcssFy>IeB=w>N!Z>FZtwcO(gHO9a@nP5m!DB zqu#?hRYUsZRr^~ah0S4KO(akQ3kcd6$X()54A@)49QZOB!a$}qLi7dK6 zJnI+a>jv>+S@eK-r7U_ujLBjuh}&c_4aBDuu_WcDHt|E%;Q~h%OhMtSdx;wyV^xRe zCQqAq^)IE9uLu{!HnzhnE=tLRvWO{pO%^dFUn(MLA9#0<9{KRP8xtRC6Q}CjE^uG1 zbGyNPz0U0c_fDPL3+}%Zfd}csxDkS;^^nx^i*Qc_G ziSEChd@<1zWf3)1Dk7!-8r6Z({R$lUpvHswhr~Rob7N*c(77=)YzOK0lr$imlU0`o z99O9hFE}>J4ov%AMWo1&st%0&OVxq8M%+W1F^eU#h~*l(M>_tRFg$k4uOQSzx!tXZ zr0Rtr_Ey<}3BE%X(?L9_h!opjRR?~YihQEMGB_@3Kk)XSg|Aa!V)XUbg(;(Pd2+MZ zbasy(n_!Vxc7+VCRjNz+!7-sICF4Non2?<^JhuZD8?)hS@jQ@z)~4)$z~#p>@qF+q z6GwDNJKhj3Ja;6z8n%|vR;Ds>6gY1dN5k)mHnFfn`r-{?R1sETaAE@BflH5@x#lIKXD-F6YD#qTiz0K z*edXZDPlhC&|5IYwYo$4;4N5!>?vg&ht5ToH1fF60*^b63llSF9eDZ#m{iWW-8)9g zdRs_OBflLJq!}l(re#1eZJ7u$Wl6rbh0Ecy18)n{`oC4DgfJpftckyh44W9whMCyiE1j01|lgtyev+iB*x$ z;Hy-h0U*IE)qd5Z3MBB*7a4izi;Fz;MMWOQVgeodlA$*SN2K$RQ4wA0y--ce$?> z!bXt`_Jf!|_AU;zG}wNhF)P&6B-aR*^c( z7Y_R)b~-%ELd^|rzL-;HGM4zOeJzoI&o(9LvqIr|zb{(roEFa2HV*q%1WvJd4 z@iqCJm8327`N9#uZCbpuLM_R3c<1M^TDYI8XIu`lEHF{(Xuy{*>MD-RLVqir?rQ&%O`0MKIQsJ57CmK2+ zlD0I^*ck9N*{Sf(^Vdf8+3>fBu`EcZ*c*OEE^ZSC!&9>oEz#<@d!@g%O?(b)7C5p( z(FoK^f6KVBHYS7;+WO@Wr0Hu0Z2`ClN5COed`8oT)La%sdT!WKg!^2Nt7 z6{fPIHV}o}l++vRPtYIeY{m zMh2!|s^nHzYQ7fHfY%X>T(zozm$)CJe;#rjt(H!_FATY;YbZ&_KNY^@PQQc-yy z9SSNZDG_Qb@i%nwt;Bz!C)Y}RgnsU5xX})`7~f==kc+#Q>1=)6y*P?>(Fh|n8AwP6 z+Gfsf@CWM)OQLcO(duA~3PovoTuYPLrOH5IseO~-DR1#RkY)&CIggq(pgS50S8c=xJl8RW}UB+wp3B5Sdk zS4)Mjq@@_Osgl0fA$|0@FsO@gO!-3S;NZ9#qfz0grtHeSxLTAsIO-_8dvGkYg2OvA zP-|#?c{G|aW)m8J?2zvNQW(%hH2(Rea1V#Z^%#wc#*LI)8I2#K%%O1$g?A5)E|Q}S zqc*{Dq(f@@S{U6$I9~i(IL5)T6Qfb#xRP9M5<-8K*77DAxiOZnV3q( zA?{kJA5`!S#X%w;-%z~NAzk&2Ftm&0fAAY&Ge`b@_v*;sA00|V!BFOG!9WV{jxC6% zEiI1L*9WM-&A}q0;+f~G(YM09jXV$2U@{6XcSr-j6UKBAm-XKXA91)`jM1n@VG`wA zHVRWHe$^;ULleOAy~SuP44K3JpsyR_2x(smYvb^&#&}gcYtfV_zdyik7BOV`CGXrL-$~$AXs+w8^C`{oTvvaU}QyCPF2_v*?iPNvFoo`3Y+PupI@Xmu1`C`@%g@zv zzDM-3W&M0ruAf;A(E!%YRcrh~i@KQx{0P**8i70omT>+n8@@MAN=b(^*h;QiPTsAn zmQ&G@CW3W@+1563Iy$>qOJNNo*sAZb#<;?A7t{QWg9Y?Jui3V8dMd4_IQ27}%lxg_ zt(F;Qihp*p_`MMSTHCpmP9z|Gj;}R<(@mHNW$xQpo86^>BprX4!m&+)cNsKOR`a|% z5K_wlM1&z`J@!eSf>eM(jM^4NXA%L61#6&f*2;pTf~zta2qjjd4yBU8YKJoha>k)r z8nI6eMH-+7qO%r!{hW{Wz|?-eCf85zDOSw0MDzT#^k^?u%~i~Y^%e7gv10z_G*nF2 zOs*VYQA9IX$tx8!XrUGuNZOTUC#nT!8Zdri$s{)ou30Agt>yvvDAza#%YuB==xB~V zDIdMFm5OKCm~~=ZD}!dCn<=3-6*K9ah^)h4V>uj?svrXuQ!NzLCr(9j^WdK6V4#YX zX?BOQNRQ2jm!R8jPbpXgbgtp?)M8&KY~NV8mr@$7Q$hW;xCfJJM`}O;!>qr@DU^5` z7|BMbYEf=$rDE!e92#0!@v=*VXQ{0URYR9rO#PxDPE-XO9P;5RPhzSOh~OFrN|@R# zi%kKfWUW0E#)&42o|$`r!^hlIAvVm$JqZwfupWlfwk45#;Trl5!F7gVfnla?7PV(G zpJJAK5}+i4>zrz)s~FWgQZfoF$@bMsCAHi!u}p-!TkUQbxdoCxpuL~jjbxH@&(!5qg{Yl0vt}+~jST;1SvLwSL z_aghmYyGyE1BX0|91`CNgXCI_MQoXPRrM%`@>kY)Ex(6?Nn(?|utMxm2j|m*z^I*M z|1Iec3WB0-?^~@uRIZlar!Hv-h{nkQy%kXKbKDeUH{(u`!pw)xj#wtayCe{FXxE@s zYIEj)I*uQldONTxpR7rcRoL~|VDTo|@{cx%v8Gb;PD<)hf3VudlF+80`ALjvIxaxf zIy8*1V=9I3fHda``^hMmW&E8DeD{yWO34p0Xh;RW7EJ$vkVS~!CMh%w!ss6 zLyr9|>Mvzf01h?Oz5)NND|pg;sACQ5&*u@)4tY>Mx9NYoD;4{I>9wp|X_W!kfN9^p zbp}TmYM}kM&H;4zoSS%96%5wdn|Sx^q*b#rx+(@oAZ`2J$*Y#fFw4RZACgY8q%Onp zN7%NR=nROdura8?etFkzWCB0RuYJ;TnEz*anBPZR8L5i+T2{h_*DlSHODBe~J={qD z#2(!g+`H&dR*>FDnOi|RN#WgEK}w5tbEWb#OoM4Yis&b($S${)e~h7t1Jrd9#-95)n%9l6IQQxNQJp8?)*X6SJ)0E9u(wuQ@ zc#O;V1*ak7r@|53PtC8#PO;4RF_|Rk_05rSwRqKD*Q$|zv8gQ%|A!}!@jIgyVf^ZP z(_#WdUUV!+nFH(!3hy3Mx?8KR#x$6i@`enUfU}HMA($x&!+uzQxGLJ>gGFFHdZRkO zA6BXu&bDziD~i#pc&#BkWV~8Y=I~li;oZT@W+FUm{iTq{#gxZubd4U*t{X4`Ca~U= z!RpmeR~3vl(T$pTBMWDK6zelI$);gx;0X2%2hvuIRt4!71cD6G9Vl}k-AUoy1E~); zT9ktMY=AaW<5Ag#F`7_$OEJt~j(t_I7~14Zf=m)%dPts;jNd-}7)Grk^El;PM&=2W zIb;q~c=wQzhl^Z4BV#?B1n}j|yu^t+FJkN_l#a_NWi*FFP0f+IP{@uq3OWPjjpr7V z`MXE4yE)#Rz$jI8-lg2j=)8|Iht5d~?;bi!INnfaJ9GUa%G|tA$O)g%Fn$w0Z+A#T z3)tW;Hd(;}_ArNucdrhUG;}CUc?QZHCYco8Jxr__Gr!&+uJ;8E%lmx25-@fXBJaqI z>09lqSs87q3i_hqs=83P$&NTl4bF@)Y!e64NQ_w}&S(NbCQc#B97shJ-aU{^24_VP z%N?tZo_Js;VT>kVPQbq2A~tkBZ+CkIf>66^;LZ*hyMkS&LmRxOxk4T2KbDPjalkCZ zm{njdrHsqKTt;%Br5f&YS%PwgdTs;Gwq1@MX}((Ag8&IwicZ>aBvz7G8w$Y5bNZ?} z6k4gAj$45qu?%IUJqVVe3VqU{jcJBf7!>))4aZj2gu-|sNe#IhLp8n#T`c2AcPiXi z5`ee6w&Mk-C(;(;NYk)Cq$1!~CU?2Dv!pA^8S}z^*tHG&>TCy|dL7o&(-|`x{cB+# z1?{DhN3U_$f-#`cn4I)^FntvC)g;+T;Vp;V7S`vPF#qlnFw2h9wBFKajs12j@0`F| z+T5Y?*TEa$sGIS##02&d>*sCx@#tFOjJBaMLpkU3*A8jM z1lE5FU%3)HKd4$IoaPE8ZJNLiaP{eFj7_ai&!8i*59B#?c5@$yYffU`FYbX^pBT<* z7)jzOm1iIvfOQuWW(w0%%j%=8G|R;)Dyt9GI1?p3RvuU#fa+ir;)pmAC4LKYRO0I>hf%^&fcQE}Z=>`L zO4!d6bAg=V36$od^jDM)qJ;bu&jZ9n1V~Dm%mml}g-!dDS$@W6;ByUJq_I=FWHMWU z-a98lR|eciC$ocn>E46q>+m7~xYt75C$&#uxp4Q>;VEpgE4)woZVDTW&$FkpbMQH7 zDjSN=Wm8!NJ|CLOa%0y+)NqQHwtC^s0jcP@?kRj}n#tA(W{6 zpF@et{}q&|{EwqV<^Lf{RQ~^s5|#hIP@?irZ2$?&KPL7?5eXm;1}VCtErLbb2zbgD zdK_+5O1T-l@)un3feY;;UiIJ;kGtWm{-`xK9Sf@gZ}p47drGmc0TEtam+-j;L@}0d z$NmF7I3_e~wV{EroYP=UV>5gXF|@(MC*CGIIBkx+it)_p91<~qn9 zQA;Irc z)Y^a&QAtU3LTaTke+!*VopD|Izz1(SUA~LW#Kb+*awK8Ej5g5UQS>&_nl1o|$ZP8r?4NDqP$$lZ`7l3c=8C zg}cjPtx&mdN`@b;Upt0XmIi{_p+An?% zX)O5w!2ALP@k{uLR#Qv!B8d572pbV9L89N4*jeK9P9b)Azm&^-7M(TAp)+Ne0`XKc zHE2G&C})sWrm#PqgDQ{ilRloy1`Lq#=iwh9WxLpb$?&32&iw$6N&rw=L5>QMXo^e9 zy@?Hs5pZx`h=8L*a&ValI8OC6^WEIPJ4VPs3S6Lu|-8fyq9Oo;~HD2lXHm`TPWoF zW6Rl>v58av)E$gAUB+g}Y`NH(Ev8s>Z275?Ej3nbITJ{HdPa~i#iC=2iiCKQXUmlu zTWCm^Y1PM}knfKz)5_U^seK*&Q7BC}Y=MjG%)Q?;frNNQkTAtUNXWwg6$#PDv!zvI3k}+@v$8D|(iygZJ(xqc zQTXXHiX6CfLVB!%O*-X{@9;`CefrH-u)(P3LK>D?hF-q7bh0|aYOjPr&_>J;4WUuG zp-r?LQHU6*4Y%@5j1z~!ETuLW-J76Q@zU~f@L!P4v{E2D4`NR;-j?pIV5MCGD7MQH zAj?nI3Bc-v_$MR66VGt^Q^ ztEIH?b-z_theE7t=}6P^3|?bmSv7ZwY_z#*9=I&T7NwZ|rFo5Pn1%QHM)pq&?;D%g z!c(r`!7v*-=>Srrt38Z!Vi%9OVC+Q~UyxW1=}b3kVTxO;R+W{-Hf@;$>>BDhu$zTe z`~J7V{`S8WcJU816suWsnwh(g-|_+Sqz(;XPBVBk)@pA%AtElRSRE|W!LR- zSlhE%zIrXnM$Y&ak>$IIoty`QR4$fYYGnhYD^{!1r;APL(~lcOQ*_d&>)E+-k8Zol z@?J{TG3VBUiUhW&1=>sHwaKnvYrQf^(B?jLMxixDX92Bv7Np5w(cjXTO#55*TQQkJ ztSx%Jn1skHV0~9P)RVA!xM|7AIi>IJKw@j4>sr~UsdKvlwDbaKkZ0#|O{p?a*A$Cx zPAZ^MHy!gj^Ac;;b_Yj080Vfw92?FEj;2^Fa1@vEJUyxLly=A*v*Ia*bdjf3aMDCO zUX@^KhKUnhWoqtvHf3tOI|!Fu!}4UN4mPFAz*JK#I!mx#Woo9DTWhj*2TV&W7nGc4 zF!!DjFio*ofGM`~bnT71{@QW+iMD6up~MeoDWr>ZjRC~kX7KNUuF$lPq&@Z8{&?6^ zul=vGoZ8DmHPI%R8K(nMG`Kz5jnaVFCeq&gnDyTLLH(gr&{Dg1J+fdI|bIc^1_{n^8Ea%wuH$F%v4@qSe4i4&iDLI zPk~vbqO%1ie)te)I7#%0|# zX}XqOFjaUM!a1d#US>XaEt@DC{32)Cn_|&5sXtWdjsNAvn%Aclkpv)pxI{~Y$c1MF z5mPJ{h=`*gDF*s{snLfv3SVPI9}4LtebVIrGoOzStHMPTl44UAFzyR&Xozgqu+jMip~PIxEFF|O{DH%;{Z)xoJQ1FB8jkRKC{@E zqO-t8tmi4!j$5?JBGG>CZ>{>R6w*mb_0$f$%d8p$EOLO(gxsVXVH@VAZXhT9>PA*3 zlW8tjEF76kvFHHX2%q96=(rd1V@;;+pyB|^SvZ%@C0rs@_M8z^OtDy?B5vnt^sq)F zIsz~ckFN3kNebyCjkIIZE!DR^Jhn#<)&u@5R!Fw~gcVv$p#J$MY~H1&^%N}s1!HpH zT(oBKPKBC{aaOnJRf|G!UlOEs!!FIcmfg&j3bbNa#+UlW4QxrwDnhUAbbbY$83cNX zDA_K`V4yZ-H-d{7K2EH;^;B!`)(0hVBP*0wV?GBItd&dqH?m>!eA(G%z52Z!3xAtf zjVUdiXESZwCDQb_oU%XTT+{uY+=m8*zDTG7Fl@r+{|)Lc@d{_GmA`GfV}CFF@gOt=SX8N zV(Brhc6Vw8Crwus%j|X|IDn|F;HV2Y1MxzJe=CTM|2`1KWe~Ta<9ULJ5dq??Q)h>MV8{xj=1+tI-N~r}lesr^ z-A&Qy`W3D4sodLwY+3iV%;=mJaQazvwg{c=XBHh(bQb7{zu;;0x<)HHR{EwDttiAm zD|_JRUmxpgt!~%f6I_#HU(flIE|1+D;xK5(@@@H#`_tbDP?r~9-C9Qscms?h}=1cyHG`dSQ> z-0iF=CZl$tU-$YnaOhwEN+0Ngi+A*aW5BK9J&_CCqz~N61?IQFm$6Kx9LIr6ulIxN zAQW%vz(W2M9QwfhKhXsq0*5|u;YMBHQgG-4oArU~Hge_o(5<>p4}wGg>HGS?Pq@JJ z+jL*{1Bd?0NA-bE83K#)#pROj(>id_r~ZZyOFYqdI$Wt5sR=~}2YJ3Zp5jv={sn)GGRS!Rps@)NpCsc2>5mRCU60 zb37fd)m4)-*ASF}LE}^{3k+d!KVOad)g@|(bZk2tnvx~`b2~dfB~!}X!Pu}xN+2=_ zViRFcvijX+YLIm04mP;_Su}2k@aH)Gyn#Q5@#jzY^BDe|z@HcJ=V|=;3;w)^Kd<7? z5%}3$9Z#!mtP3>awU&<@IPmNaHZ3Kkrl$McBf+QlwG|Ao6)H9jBM z#V+%pVri`?6iE~AWfRc5@m|&-$5)sqefLz)Lg|isS(?;mH#-}H3wEChg=Q1M^(usnQzvxnv4Gv|IbL4K`yi!fALdq3-g z-Z$URhK?ZEGV{XzI`~hiz+|`z)Gx}3iqbVv3N%OVXT#*k0pr}YDR2kIUOw_Psbnu3 zEypzUE}=lWXRn-~gL~Np7~^MqSrIT!AAH#rykY40=8wKZ_nK`6GA^u*tmi3v delta 92927 zcmce92Y4057q2@>NZCztw}qZU4K0NhI)oMoEmY|p4VT`e_ZkQ_z#zTZ=%{!p(rl}ykilk;{BDdW4bfk`zx{b^t{}RjBw7qczI%OgQaaN=N)ODNTWxD zm(IJG(I{_J^Fs7`@36vo2b%TDYuKzOT~nt(Y+i@vad{n^JLuuJLL=L*AdlonNq~e_cC4YWlDp}M(k)TJ;PiLB~sp|&zriY zj+DZs;Glv1E4vnsk|qSxZQDsfdiH%1M$3+q&eKB2LgD8$diO(8in_*2Eop&wELs>X zIYGKg*DVTmO9WL@<7oOs=`A{?dzhQ%tggn=YLggW9li33o95~_ zMGBQ@!8d|UOZ(iZ()V;vrEu$~YyLDTR-)G$xPQ}+>#33S^jQ+>`f9pVK%&)qc+NbY zAzh+>_V@fuYp52Y$}H(PoioToyLFxz^q*PMK{{)?hj!%8V8iv(Y)Ky>m7&MJRQu6a zzEV%n?GL4cQX87^hg6>~dnApMs?trHLn_iGxl$Od|5zGCH*OB81V3F@A4{f0ddW20 z2&v>M`HvJELb7FgJTACAO^FLBtvx3{^GtT}?4<12R+YzO7j^cN^V;rS;as zIiPcfqR!sdx%gnxN9svu1e4#T)m(Ksx+#PdB9!5tk#yJT5OQ6TUgc+!>D5po^E3A* z7pO|(!^lwSn9tgXCTl@=QRmzAR2XS3y-k{;Ut_Kkg4=QDldC^-jeC{6H-aK zC!9=W?d0`YZt6j~F4JKVq@#2lV0`qi8I(+_5D!8Znaq_U=yjRYl?u=}g|uV;oLd-_ zBDc!$U_1^P)O(_X<1<)tTLC@<9m^;Q&F zB6R>LIx(7rOIdVwH0jL#;TF*-2Z@kcVT9%$eQ(Km_gp3E&_Tperij3MI!fA?G;$XFU5eCcBaoZrry+u+nI<5q1Ar<0WmFU?RQXn#ORt;vGrPo1&OOj`S zcNj^b3riBY@)PBM&XJExlGP4UJ(8xzk@8Ou%(7k(m?~6yIjA6UT6toR3@}+1rnnh> zsgfS9$I)KZe4#r1Iha*&SLglRPHp|BE-DJ_2vfJ=Ggvvb;Ghcq1)X z*^;C+cDsSt{x58v*phrgOSd2uJ61%=NSrkMwT9JxR?>oOXXKIQ%_S0>aZ@2#>wA0FxD-K5jO?BW zJvH;RRrGEr66;PpdsAwwylFM>Uz)OZ7jh$*4UBO;NqlHi#_Y_9qSMu|qNQ8%ANe&d zXejF33XuqMW<}A2YNUA4j!cFR?(SG%JA~F|F>~}HX`ZY=e200>G3d}<kVRz zzw;23;xnsaUT-!GBKt_|Xz(Voyid^Wlk3D_(pHlCTB@rLC27(?P_0z-@%DA?8cNbR z7dL>$4JVPLm!&uov2~BOyqPa%@iaO_8=zqV!(C50*lYB;WkM>L(RczY@?z2w+y22DS{~)+HeP=8=Dt$n+ zjs;buyT-9<^lr2@!M`|;oRG+QD-?85^-x)Q*L8IQi6YXEmizK#HV?i7?nhSt-rH{wY9X>o6EFix-#dI$r8zA>Nwh;^B7u&g+^0yz-kuJ67*QzP=yZ4A<5oI z?am=rBsK?O_En_U zRI5XGqNF_uQ|a+hqze6-vOr3%o0LsdC5bM*3EeNl2l$U0aQ%y3z>==hFY(sf0 zEFvpaUN0-q(py=-%46EU4$@tpzO3CPuJl)k6hyXKl1e+D@;o|i8(B%-c8fM*t0Lrp zN;mBft1`7ux0Zc|KHn(2&%mMa9R`dW9p8TV;DMt@4;a}aF?;mr;iGCLCl6)}{WwF? z$N_!&kIo*@e^|d!Ndt!^4;aeMjUF&+bn=j4{l^R*Fe-U8L+CZijgXW*ddT2%Nz3`= zV`BDX&-G+(L8P%JRtcQhF#>~de)e+fnv`8TsZ_cWRJnVT|0hN+XX9D+r0m$tXMI6C ztJ0-c*v0BuPivxcr>xeDM5K>dJY8XMO|2@3et^f`aQ(HNyicTxu4B8%w?Rd#rFvWq zPOZs)Sbnhr?9}*FCWhW!8XiZoHL5NPFV02-;|)@ZOmLsedV?hJbBmURS0&GDw0^Q& zP#RBM-=Yw$e=Ux71Iln%y#A%P&rJS=3KkCLN z{LT^l=paAJ<42!x*?s)z00VR0;72(Q_D3b`&k~jWslt!m;zu3%(LelXEIS%AH8*@{ zpCJQAj~_l@6kkJdhHB3|ljB^=&gD4ghVsAD*xz~1^e}$Rt<3>rbHLDfU}O#$m;=V; zfMGdcR1O%F1IFZlAvs_~4j7O#COcs=)%KAVUBESd9dEhDuUswH_~n}A8lSvbu3g~n zf@|+`cfqw>?k>2N=W)$>fE~H6zD`%aw-|T7FSQuzI>&67 zeU6=7`ziT>PWzN0ZupF(NVL;!#>3VZKc{ETv(uM9XGh0AXGeT}aqjgC#C82b(8eF5 zzf8cBh%$8VJHe^0<41$%#JT2Q4Sq$EYFbfx>3VRnh%QHh`I_ys&%l~3Wox$SwB0ws z`=#nG`9^R?Ft|jA{t$fD@~G>N!PmJ*ajvdE1t&`25?y~Y_^RcRC#nVL8@GawT5c`+ zIrszaQ5+rhOYjZLqmj3R%ep;U-B^yGNACo`%iStYH{501;8yz#xiH=MYj7ytbT9a0 zI^Z4)AM`KREbu3u}(A4qg}Eq3Z!9l3+F@^dYb{++6p za!sozmzU_B3rzmg`f?eG*8Wnf;3}Ceca-SE28@4WL-|vQj%&ycS!*?vXpJjcIaf$i z&U?8Dqq3&zF41Dw7{V9L+p-hiuRdx_}9-$JfXeII?B4z?lE&ieV} z<<3UzA9{S7T+nrYP4s64EL?d@^jg4GUHe~(P9b#ZrKqy5=vSjpO|o!)DdiYN=-C`b zomNd@lny4o5&YRHHWUW_D zjiMQ;ax?mQIklynl+D67hr5zUN0iVShUD_QRP?(eS_gV<6kCD4R$eV2S3q3%%B#2K zbVNAbS#3_^&&nZ&QPw$=L)GPHXY-?a*>uA(&7^%Qs+Hw7qEKVnyQ&{V+{!HL z7NUFWtHo)tN@@iYY39-vJnddd9U(t2!Zp&fU6y=5#F><&u9v5XoFzz|YCDGyg!V~G0nQFXz#13=jIS&jkInp7ddA}VfmfqbHUXwnl ztftA=>_8Qr9-zfl)Y{S&1ay(RP{R_QT2-weKd}oh-xXegvRjGC(qkm|Czl&ezputb zVh6H-yHkVvzj6{(8kkg4dke=Hg-jR+;V^%h>NzrM0nHakl zSxc&GiPf@+wK`x{r#;_^2$lQVWv47~7<5{7xgcdXE>oa@O1&(U?lzP&wbY98ED@kI z%_?gqSWU+6bjGR9={zZK@;{KC(=DtiHmu%t)ME0>b}S{sgT-!>vT9mKbx2z*fQ>$Y zr3b^)R$kB|q(6BUHMaGS^GKgFeU7TReGjfpaZ8US|D0>jC* zYp7OKCQhM^O4D%%!pqSk4b(X4i3Rr94TeSQzhf4s4I0X!aw)s~qVGKxv0J){QUXm# z_jHR`e%#QL;LM?N^z6H)y7Gz^DmS&`OuFO2>3u~jE;T|p4FsHNjagu_?cyn6syi@z zppyn5@xC5$Z*az0!5IU2%)=!^1#qYC(x&R*SYMGT=q@t2u+W4m5y><&L(P;oAxl_2 zY?S;G>o{W|Z&-POGF8N{%B}YaxzuLrkCMFGN1@*ONOyOY!%SasL}HYQnXXxxYAvE1 z_Q__`wp&;nkIh#b$nX2ebkkL}HQm!r#b`*3Zfd1&5^;`y8a`OZoE+;>!`C)=nciu(c>>kc5+7m=lw7WMh;euNZx z?T+(B0paja_jN91-IFd_EPbe1ofI>mctj5JrM?#zMHh+DQ|;9GGA5+<)LPNq?bUE3 zfScn~d-X#(5our{YD62PJHiZKB;wtXC_RX_eaRR~hjdgmsXCq3QGGtYf8w8IhKv?* zTayuk=ya;2lc9sCR9#7;-LqJGFR*jXvvJMnqSli)+qpK_xN^IyP2~f2uD5Jl>$|DV z~)5w2MA)i~ie-oU2Vm-GXKLqV0(b5XqUYKv&FK4m z)pqh2JM&0jW;a*Q4OBbIx%SgJ=yWpuygxH)i=Ag9;z^;O3{XqU2kksA#FI=9KgV#+ z*?CR_4;eO?3U)iP*`(^8o%yx}c#GSeLZb&VrC~#CaSx$;hqF=+JRRjY`{`onbP6ps zf~8H#|t)$FR&uvNKf>ncBU;9B5)^YUpJW9N08g?H~`ZGxxQadkuva zSm|Vdlk7a>1s?jwcsBLQvokFbnYK@0_HVN@y&^Jgn#j~0w=*5_G70wIoup>T*X_(- zTg;b`{SWOtzY9F{pDE1#=wV_b5oMTFl-)@5!Bplz89P@=8`sa%7*<_7S1lix$AwLQ zMW*u_dORa4mG-{p2&E4PI5fh>lH)%qCS5l}?Um2i#O6IQY^6SNrkWMw8%F)-Glyrg zQRD?=3ZKuA6WR30aJdk|9)(!gG$e*DnWcWI%<@t3EF{-Qwq_wobA(w45~D5VsQYD1 zfKj*GTs0iojkL3ivVn4j>-b!?fh2E3ik9YSC1_F(b7wE29$%!kE}rT)8N-6#=BT?B zM8KZBi=g=Y;a{L$SfEZ6jN;D3w({5i3c_WVg$)DXZbdBYYBAaMd#>6+lD|b{aQTHr z%;j5%%3SV*S`G=gDlTT1fC2apP#*Qla^^(`oB1GENTvq`tIV9V(YYYG9K7liwsg z%}-|;Bm}eVODoiSl9J{lu*$#2uPNB{z5YlE_5M(7Q2}%H>(F}zYF2TsbUznAMW7I8 ze-><`w;<5& zk(=2xwM1kUI{$U%?66m&Q|Y6xYgowEElLeKpNi zU*AyYnOK1j2?}?kH`RGT$|fIPHihMlhc2}xn;13OuV%I06^(2` zTqF3>Mm0L~fLa?ZZFF**fS_HM4yYw1Q5mdl^!S?f%2P-StU(-pTWBr>M2lWk;{)Ji zUsaRjtN$Ixb@6RAoX8j%YESK>!n6v9&r0c>qb#NWLW+1wOJ!-&F|{zN|9>D37}-Z2 zQ-@1(*hp_@_AO`?@9vjkSe&Q7YvViWa3U8&;#N_6pN$_S5f#Rd9)5U8(DldrY6lU@ zDhK{jg)#7l!uS@0PO%;oONXzE&}i02>Q{1ObP9~@^f6lzY=Nj?WKTaMv0%(e^^kyM z+2}8nj&LDTi$75-N%BBU=JVd4PN@y$;fM-G^zt(T3A#3&R-;8I%L;#~=MXTI1b>!( zhy`lS8)wzGvFB+E>Hp+yK6%=BmRGxO)q)Ia0{ zND$0)e9q?7#}F0F?Bi!97X09I^@@PR&7_Ym2&4U52op^7$HXRji)(TJh~w0_{zbm& z>xH1XUkYUhcS2q@h&PAedrQF0%={&?T_ZPM7E1LP4ejf=z zE%RsbvnJ$$x>1fFEp+(S^q&WUHO0{RqJOAeWlVt9IrWFoI)#w{v`(dm0w6ZSTMuLX z6BK2(yX!xCsBR}Db2L4d6kUfjA4S!isN$&Ow|m#>_!-Co==kw}3l=s;=UK;pmfn7> z)<(S_X||S1I{d8`k(5C`F?YG^QA$gaNBPLsx!^dx!!X`i~mK%{tES6nq_XsTl#cCZo53!7Y zs73P)g$9kEq&w{F$P}xvwxP=+w7yR*tTknAf_xh3;e|DIkiD>G&S2XO?U6r;#CU0? z%@pl`a@j|9w`5YJ)L}?Mo7o^=Ccs*L9 z4noEVDU1Tow-{Q1e}asGz)y=80{7 z8A+#>)(XhzJfBCf#g@`SOCX!vEzzWmHYSSn$+so=zDBDhdLj#bw(B=5V!1bn7uA%)! zHsHdXmi#*L-fPp5au$hFVd{ z41Iz16sVklI zXquI!C85SZ=}gs`YC*a!OKXnKqY7$u>L?o5R>KzzXuZ?DVtKHwR$1PNt>eo(S7_~a zT2TbO4spN`cD|j~l%8m(^_F*`Q*edQAR#h@PHQjZ5tfH5gvDdEaM#`TnoE-3K>%>A zeZHfXB7cOa=lZf@+M|17-$Z1vbXYenOTLGw+|oF{m(s2miMrCdYj-38jG5aRMf(SY$c>~|dkS*^Y@RiH z?WL7Q@;KViTGM)IM`UxXudjO>Dvs~pv>S`%VU6jZ-dZJ@bZY)#ZiAR+vJ`?AbcuhD2s%wol9+nfXWzrcgbHX=f#Q4I+EfD8A~` z0o?cKZ;#Nfv8Nmn0<_knQS1#nZy_qQ7T+#ww+V^5&W+~P15seo6&FZs2g0^mD-hRf zJ}wtauG^CQg^xnP(x{8!1?Y_NS}pl&bPCMnyJ_v_V)?2Qw3-ScxaXhnkmg+k#wY#p zng4?3YBy0EE!uZ(lF;|gBS`2;8#AKtbp&#jKWd!pIz3ri&u+z!^A!p3_{FJOOSurD za%XA1X+pO}Qm!7;w5cKvbfQ`T!V`sHt!4?tg)N0nbZ5O*ofexZSYS_9+IW`s{VQp+ zG>HzL${wp`Zyw_N4Pb0qGCN;U=zpq{s!JnvdU3W^70o97dsq88S~Iy5a)S3RwoG?( zjxcz(M;y?RQs!!!B=<*TpK)vE36nh}>+VA&hsSpu5{7o#HD7BgPeoK{r&bFDSCN!! z@B-~Q5eEu#g@uB|7=#FEv|yo7kVm5PkVY35YRQtk5s|^X>Wj5@@+*i6eTMI&w)Yt% z`qmQGasn2lQXr{y2--@eKwN&-YRh;x`Va{-Yp2L(5CIJ3`?KwaA}QD0b5iA6A;mW54Gs$&nB~O&QNDP8mWEIQ1KM)n#`p_yt8J4=ZO1m$>Kpgq@aJy5# zAXF=kfw*jOOmcm`T8owBLgQ`4BYrJw^*BTYLr<<148>A6*J_Uh7zks%^@4zj03nR) z*XJ8XPr+z_22#|g-~oEtW=vFwtLFxdy-}eF!uEE`xK;rog;99fkgCvOo3*x2Ef`T- z*lfE8QUL{{4PCrNC>UK42P+t+>--k&9?@0=xgSPxI@$UddxhCj*SRg)eNxE1joiAd z8H9v=9`N1s_O`}qz=Mvo!4A!!*BY9M@mT=`{43zsI(wFQCejEFjiAo$fpx_^>Gm(dApP83Lbxlfw!fMHicfw-26 zT&XmrmsT_`5GQV9mQ4kDE;U8H4keE@ea)q>@6dW6jzs2#XEB0Vlu9=?Gb5zBuHHMf zk`k$#P0uzrZP*u2LtxSX)*t`a|j?Q?DT z5o~}P!H@V}HhL&UZXfE5XOU$uV&MC3qoGfXfCgrJoWfz#JPmo5J>b#lh*n35nZQJ0 z{mwCgwoTL6Qo#`|M5@7e;*L~w;cNk;IGuJ>dtE7q$$k3Bt-L#MOY<|Z#j(r;z5}uhqmnvYX@e1^m%}ra7M$YY zNlsG7(xZLML|W-xEmV2V#+@LseLXHM-r9q^;9ac>={JF<*HF_*?+J87Z#l!-L;L5; z;YF2MSomCPxfHEXn{7WF^a^_!$#l2WlnK^u)QJ|i-s?}c zxCJ-b1fQHVH9GVJv-v|m=_(&E>C-2);^cj|^gBN3gp*n)x}%d`NV#bfY{lQAQH=cH zmigB0Anc)(Kh_E<;S*8wgPqZA`z3$bMsfBY*L%lw_txYfe(c_9{0zNb#c*#B9(z(N zsZ{U-nbU}Y{B}|+%PyOIAf@RhldZ497Uk2QXmLuWpWw-p%+IE$Sh>zXf{l^(bgtdK z&pGWgvl@+B6%nco_Ctz2#gHzY)*7(4#rRBk&L8RQ8Lfe`$WNZX%YhC&s}*B!ZS%>` z6XcgR(^KiShvA{HSGF{T-%HMD>}{r&mB-F8?6>`}`O6__mroh?0fhY)!gk9;sIqp2 z2h)kK8M^YdA5z(x4C$TE*y{Zy3n{+7%65f|ZRL*Eov+L&I=q(=sys6ZwK}Bru39pC zS;QFZ)D+L0q$hzKAKY%|Pu+a3lq$Np<5WoU2fM?@1x8D;106p`rsqsz9 z%27Yeu?-m}Y*2n1VIHtBJ%!}f4YLqk|3D2@uKNL`vA2Kl*U#wmyRS8e^ff}cB%)N> z!P?=;cGgFNCfi&6QWN&FkD^V~Vk}?NFS^@7`r8ZOb&XX2iwGkpN1(5SqT%C7F!i{KO^TGTWVsd zK3bINs(V9wAy~;2=xkh~%8y!D%+mzx|EN6|q;$rDdv9qS^OH%a8e!7SKWqKzil4P9 z%5zwb4}^5^FZnBkJ6mUe(dI~G0(+CM@Z=YYo96+Lu^2qzvDyb2H5#8gw_5MSpZ?!M!40mu_q4ys=E>H*wVcWD01DF29=(7Od;`|O^mn204+Q7dVg}}1 z8r?WZPo#%m*2@~l`8gQhJx_A5;d|39y#%elAgYP(%My<^w%P8YmP#9BMk{pkeJ$2F z6bOMmMa?SieXTrla6dLfYOHmpaSt>c0m06ttK?4fm*2J82tf!DZzVeGftE%75JMF2 zH5xrxavv*i`-j#K!QR8?ppn>PC3O*lU2lf#;6v?wLSm-~ZG#lU(;?70MW>+NoLYxH z82Ha)w(#f6^+K2dw4Z#OzjjEB(2@E9(j;HyzQ;NmVr)>(pVmOi>thCXi#f@)JV-_Y98d$WEJU%V6o4vkpPsI!n?#7CqZ)3A?Q?espy}rjbUc<1Ump_Ps z^uCN4_|1IUHcGD$^FAj-hyg2~5c~AJgJB@VPtyY-W)FzCAJwt33UP+=oaN9{i1Izw z(I+jk6`fEh3i~1mvH`~=c@wOa^pd(>89D7gr0`9r%0E~EA5F;a3ADVS7eS(rFb|aT zVTOKCQTU~sXEfus#h>R7khtT<&OsIk!t7e18NF#T;dm^p{vMt{&&B99l(Lw7y?Qp2xdwMib$eKh4qt4 zGfZZdm!M^f=#7*%n9MDY|1tokAbg{UzFUL?TW=K=Y;A&p6CNt*v}G}Y!Qal+#q^Jq z;aCnLyrj6^Ss9DTVCTgEb_&9MV)ZQ|9N5_-PB3E#1_nEihzx?A$6H3Wh!51ZLcTyk z@0-9koVG*>>7UE@;`PePOV|RQ%azaBXGWQq130@G%R%ScpP(la=Vbf620E=#yXBI_cGFbFY0E@7& z>v{!Uvf)8m)~+NN5`vK-E!T<+1T(O!kV>wfD(U4VB?Z&L(gw-;a3u|s!O|N6EER;y zrRWbtIB3T}a+NSTw>CZymtZolOs=w(^-xLajHSWkZdLRdN-srUl$OooX65Fp}$Fmy0Xv3IZ` zbp*czgZQ}W`n--_P*Tod)nHNKdU|&{xs)D3*m@;I^>jS}N)WkSProCAK(hW6!2VCL zDoEBqTptNsd>H+%f&QEF2PT7^Ar18o%0HM4k-Hf{vmpFwL;Z{h2YK*UBf-i07#Q-P zTVsL2&ra8CjrABwiJNY(SU)$>hbm<-nHyT}Rsch>u(!z5=@o)5xZjwC8Q3i$R^9`_ z6vN6=o9QjkU?|wl^Os&}CO9T!3Z$;qoTY9i#)Jl#+nl9tTTF&*$;=es1UblBOcArT zJ+c#<;gVYDX_7J&>jD!7v}B{!7)(CiQhyeq2qHJc_5NYttA1erVx3W<76WDtym25-?` zK){ivyun@5+UtKw%5jVg6ZT&NL`sl`3D_5GEu23aSG?1|Tv06Vq^BrfVJ$EV`^|3_ z2G4xkrZ5+<9HeQNXZ4%P510&BKx;b--imVk?SQvBv&rB?EC&}h+C(~}i*RA{JLZ8& z%C}wgPLiU`uy^_AyXkcl9h0r*?k1Rm-B9=kexsB=iP0-hht>O&OlrRFc)NJ z-=6w*NvVqIke#)9>$Q~Hm<;Bg>MfXyN;~y$l|7gYeLk4)1F+?vAk0?tGF%@G(w_|`@6E_>Rn7O9 z{c)@TmRsPNe%~_B5zGL~JnTVxyv!qL_pPek%2#;_{!B4kME!`dLHVQ_!urT}m;p9! z+!&>YlbGO{tc$~94}f_tWo+wdK1CAe_bhh6V$ahQ!D7$h%AwKp<90#r-9R>ef2V&N ztGDHMcZTbAiF=6|E$zY6!r*fvHs$y@9j~93#^=QK<(N#b%f2U%@d~pw%0f(h$}(os z^!r+!iq0?C%B+>Z&f)Pa7GO%_z>wCGbGN7xG-Z;|Gt;qo5Jek63iQk_ z6Y2Algc*_0i_#4_g+835k5y)3<=HW{Ll&!-ByTYdENkmyHoaBpf zi>C?h3DPj|zBf(RBxM5@Tw2*|O;cv*c)48=ghetsL!Yj^f#smEH<`&s*n^l1HS_dL z!45(0_Dp@4hy=q>v&bO7GOz~&L*Wd>wNv2Y6Z!$O^|i_+Oa^0<=I9NTZ!nn~8{a&D zWASD*lVbN*lLcJ16oJB7$)XOU6@oBc`ltd@*kE}_` z%;Im$sipHT9_NOS9PnKKVMEx*c9uJ<&az&g^VcH17^)_%xJA%~1%1^Q>r}J=A`jfQ zAw#gAy;Ufg)?cbupc|Lyt?2Stv#gTEE%nsZ_@(+(r7yO9)7t1vRB@j58k9B5daYvo zGBzY*EeH-4={7FY?*}R4F(}Lre_1KCErzh(B+-HDbBVN&9v>O7(zdJWiOqbB4Wmof zH3Ifr%)o}WJd`hjUhk+Ty_uidzTG@(_hM8~Td%z^sO>@wE;gtw4vFJWg+Y6s;^&}< z1E8k?Yy+E5*~bWrLP>s#G72Sm%2uHy+a0CS{rr3!Im6@QJ*<}91nf%dzbur*+X91d zr!+>N?eP%+k&o8BQj}LdjgHw*8_6OAL&8YrcvY{68a9Hn3R8i4FAG^Am<|yOn#4ZU z1>53ntU~Pgi1A9!Yr3Q~#42FcHsduGIk%%sJv6l-45LQocGk^0VPSXy)`RVOqS6zS zHz`rA;%5cG6olh;vs*F(1H72<|*0l>2aCO&d}u~TnltSISz#yVF|A38dF ziO-dKbI4n~TZZ75ZeC&;o)Scp*@@7j(~VL1Yrfgpp+Szj=_l zN4RJZngc4--aWdLDBCeIH0H9i|I(N+e}C-_7X1TQ7)I5oH}y>A7$(E0y7>=(OhFiy zYA}VRk*DW8c{6`cU>-AdpZ-8nzQ9_iyM%=q!F#YjXutk~@*Nfd=U)zBs2~iPTwQ)-1q#5i7oTS;i~e`GdeKou1HcVE z!6Lr#e}^mkp`Ikd_4zRWZUi;4xTJ7s*v)}10S84SAenP8A0_xERdeGv(gB>p<9(8Vwk3`T#Q&mXDC=lXmR z3A*~<-U`*)hJue6hf8%(4tmY+SJ}1mo0tsEw&a?iPmtSl zjrBi);!DyGFcK8T3jw;2Abjq+o*}|Pu@5A-4WqNiY4G4FcnsIxb#Dkh2>li+*w!0* z4k35v=2yYK$@5gOU$A0U!Mf6&KL}S5H!%ZLutz&QQ}G=?ZU{r&dbk^ zUk`fhFy^5m1C{j2PXf4#8DJT*-CFXN>bWlOt^x+d|NPHXd#%|sdx19N-r!2Nt=C#jaJWLG9>MtI|7^_ z2c;fU?4@1{5Ahqo1%sf}-@U7!Rc2z%U=Uk@Vo!44V{2yf@i}f)yb%CcKzQ|6RyhTV zulTIRaL}Lr74F1u>`@E>4l2m@zl0%o8U}_6vPO!=5ByCQ82B|*+u!x2%3(|f-=iO} zDaU)53~t8+a9a@m@Bv#Z6)4c}1IZo4=rG9(#3fkFD?q@|3vCM7`Qk&pv84QnbgW2- z#`l5+5ttkf`b#H7d4NQ$uTZ@msir9ZU@{EUe-k0Jf?Rro=-2>Ai~eRQ?#Myi{a=x) z{ljMW0yZ@K`XE8M;0!E5uA4~XpXiOzY*<)=1fE(G>Cz{98^q&(M+%&;iA{!O%J==2 zDNCgIceUzRjyKD`o5m-Sl7Y!kGophGoTh$quEs$|h@^DHg3zCKh8dYk4@`#sR3?Bi zf^f|cV}l3>c^XI#8{=wQz<8M`lk5sp}F?x~JIedRZ541>I`B~N;FjL1OE;dFE4k3GT@*9>KPVtEC z#P-0j)G9?tx^0*NhQ}r6Jj0`n6`No!qi!*beT00GuO01P^4M`6s{uQ9R}t(uhZ(?* z^`Cp}u(1kuz%X6Euu)qg|Kw{&-%mVtJi=%ls@MQS9qQ>!QMky0F=vewASKN99I9+B?ZnL_~rv8OQ>`EhuFtC(^%57`WW| zm$Kfk8A-|*n_&$r|KL~3W)vyl=k!P{2a#-E%GgYlnV1fDTnd#nlE}jOv{_lBys{Xd zhR1vt%(ou%J-MCjiM{r-67G4{w|Aluhx+dm&k7z_KJqC59vP$SyrL-xEsK5rG~(F z#v)+ds;Wj+r6wjrOmhP0!@|Cp_W3!ehWp7x%wRp47zidAD}$s6#3eWYQ@}u}gOlBL zuDbDyr1Ze>*4laczw~SvQCrqD#*;w{>?7*tY)_BGx+oNbF13tBM45;cTJ!z7Ms;N- zCPP_$eX+kCg7CX_jS?apTe@pa>(?{zoqxd&HnKORTk9F`kxdKo8{OY|%j3d&Y!i%T z^%@9Cy#_PDsQP9D13!2pC}(47hO2eDF_e%G^VR<86_57!F}O9RHx;zMgBf5fz1rAF zjk(UraK#H)`HZDLJzrxOxIel3DSBWmozygcFX77K+onc_q>xJHC9oQnXeDT|Rz_*1yg+U(lomnc zU@IfV4BNzsH z%+vHhk15(M|7b&ToY~EIohaX6WVq^E)Kl<42oKEaewj$$?`bqbGe-YeU8`Pfn)MgP zhNrW_C(+z!6>miR1M`4gN^c`KSP5Hb@5vwZXFb`0$xyT#-|ojCJ(Yc!Y+ZBpXDGI* zydXStobjs&#|EGl%8CCSsmTOmzla14I5L0**dyx_{_sRY3Mb#O^x+@Y@PFuZ#sW_? zzQLA3@v1vVi11a+07W!nj!;wt?T|hr=NOYDrNAQ8d95kl>;BObq@e*P%`;jNW0Ac9 zH-F_ZSrCD#)uJ5ZCMmxtKW|@uCfwcjWwA|Gsa_y>TM9FPw{xm{hK$xCtG^jNa#=NL zo39z~l=Ya=8fyVF9E$`qT3`k+#B8$6#zwK5>bld?aQ7 z<#%g(VqjwxVgSSDjU`4UiEPN%j&H_z>{yG{FgvWEeI9 zE1zN0r)M9A*}L4pk5vmQV8ooZBL6tU$n@h1<6EM9g+U>?e_hKQvy}!RJTPoVPomvl z6pEfOYy!`p6X?e;3JWIw`=a@)=9P!o2w3|~V{gY1cO#JZ{LT6u%fS<`Ro57emGH%= z!vb6gKj);j2kQ|R$)BPnEB<$ZO@3POW|LIoy!0qe9 zokr*I6~yyp;H+K7Xr&}Zh6u$ku_EM=TC&?XNRu`ju=N!<(rg?tdyQPBtshWp9~*f6 z+hTgIozje+eBDS^MqqYuwCo#34P_iAgQHOa938R5a+LiGjt;dnNrPzWYeq6%^QJLJ zS%s0n>=h!{FZ+!1%5Fcd9U|94moZNHz>n))FIU2LqapFQmUO^KQ@+RC;M%eSMl=$hqP%ps$z61fy*!@cH8hGJrhT3+dE&GevhgBIwykxS{W z<3^bP@`?Xh{+JO%yB=mQg%gZ8zFlcWB9=npNN!VW9xHb)%Z8hw8$r5_p{MR{=Ko~UL2N5)Ylc9}hY%V0{rT4I;c)+den z0px4`SNZ;*7_~)tu(IefD|Owm7<7uIZa0hxvFL{>`C8d+8U5$xZMuB4bzKyNjd1H@q)sOA6jqUX3Y}|j~$M&0#P4KMW7e-|zCJ&_xdmgWsQXEsD zlxhJ=X-uBw8T%JJ)AOu>&5>t$R_7~Y3H|tEG_ck3aolUgM*SBr8YPtuSQ+@<)}O7E z?8lYu8ea}&tu*uEwpA?g(UNOd_t8-z(1QXTnljs*$jMj+9 ze`e|Ynl<=VYzWLO$N9}HwOjrZ)lFEA-z9SmzG_Gk9dynL5Da9iud&I{-A21nAEEvE< zLHN6GjnyI?tN{d)yMoc-eoY`Q!Q!^-*=xb)e{Wn+!j_|K28$Q|z$}i&WU#n!0E-3T zW@kr+u#-OqOWQc~5SMn-U>|}{!O~!D-&^eVSQ;jCYw4X^ zf>tc&ivHPP8#pn^y1_W%7eS^H1_etmi420J@ED{mNrW7RZZl;PoqXG9{*-Q$LX|s4 zdu0Gt&btj+J&EqVW8kIJzL*CtQ19O{mWL>#1Ui3-TPZEV?*bJ|yQBw3s}N;67KBC8 ziEK9_S+JaLf5et^7vj^fNSd?US|m+-A=2jxef(nia^^<+Ns~nXFv*Q+frIrh6VN=oDs_`geV+ z!_2j!J_yX>@caYAie-lDLAcp~y{Te_y>l;@%~Eu(Y}QkBd=C0dtRjG7VOKrHWP8{! z$?7xPA_bXn3<`ZljuIFOUpbA%wAG~>Cmy?j5a$cwJ{mWP^AE=jD=m(9VUBi z7A9GDx&sNOVt7e2@eO)^OBZWqYxdF$ECZHC80KiDA0~sP$pI|I!md9Ilf8=wlPpUE z33kEoP~!t}2{uDTvbB%UgJ8k-`GV$fNtuh`y*)@;N)HB`HI+PkmUkd&F+Cf~zC7}% zu-Jc4zfbk{AB^qo_jKy3!KK~zMlb_~h{hB(*D0@Ktw)QRZIpwU0x!{jG9e;7Aro}j z*53r_xy8)VA|9K-HlkTuA|v7h2@4j%-3WmKBXJ-(A-u4G`iJ7?PWsCdw#)l6H{P>i zc_NlMcpJN8+dEy`8@|gctSQ-dapvS0tN_9B;!oM$>0sB=c(YNE60#EYduUFnlv$5N zucYavSV2_rY2O-L3;EJxa*5HPh7D{A2uZ@Df!==sH{uZoU$13F5GG;kaWhLEQ z#;l}NvsuxNq^_iUH#iDY=bPc;Ls&=3nk7)J%ttXalwb-p6mZ+J1IN*Nozkm`W(i5@ zifw|mUG^PUZ?mzW!bm1a!`yU1d2=4SYi5_GgDVQvZxCjH@_0&Q$QBsbw9-|yl37Ml zW@9?srCpsQAkV-Ifc!{g@JDW*Y_j`=Yq21B+$4ai3~zxu;(xeyQ)<)(s;>=9fgdWEYIj3=kZD>9MuULH9ZZHKukmlfVD#Nd zbafM9R^T5|{^>5I16BYLRVLFNO@)ZI#XOgrn%%002YYVWINy8DQ(Q0`qxlLKte!UN z9YLpG3pa>1nYVpydL%>0WWhf0Ve|z0dxkjxU2-_syW6~vGPAcKRHjLNBf?1O39ddn zwGt9PU|)!emdIRceg28PQ3yW&)Tf0RPbd(r&p$C%x`cldY7V9Kli5=LN@{%oY8>}< zG}lv|zx}-m|MQT)aypOGHt=`V?&C+F@wc^o!H<6BH>UpNM+^AtoeHtXf1OqMquaIk z(PaKkm@fP%g1oRj|1*p~6nb(Z$G^`XEcu0>zQv)VS^hW)H>)l` zTFt&-7zP5d%$zz&@%9B(FWdT3-}*s zBe-!|oj-X{g8%8l|1{x$pxvMdLc7756tr7&ev=B?4KBD@yG>6^D`OzFTa!SgwOTxo8V6P1`l7fbi!wKR`VnHKF){y2)=5=2PY?do<`vr z5;U0}NRd0bzd%+KzCgxb)Z3=3nIg$_T37Q%kqB1%`3ru|t+M|8jQ!hctK>Yticaok z&WQ+fpFgpR_UdVx^yRKBliGJT6Un!$=<%LrOa5k2|TTDE+K$WD_%1Kq0 zj}cL?6=$gIQ)p$zjBzRgY@@jEE%mw4{f6})Fl6A!k;6vz8o~~jbK4%fHL-s|Q}aky1lnqlr*%+5zZ=1O^@X*g8R_P zz0D+=(#I@KvaFNgo|A+I^){3F_tcU|PwQL|J4Z|OHA|+;%u{C>hT`On>a50(c=I_M z@*^H`XBI!=5pnipM`Ln>M-S^a3S!KKsd_KGF#ne43^MUWsy=5nr49C&YFu_K=yz7- zGU!W|a1k=@MYw6NyuhJ>GU`SC<)|8DEc-V}tw!s;8eTTcIOdv-v+`#9|Lg7{r6I)31+;Y<(hy3vN z`{RL;S=+;lyN&$DhicHjo;M4Ix>0G|L1yDsKg`Sim|&wa&`e<8ZF4+lX7E62bnZa2 z5Mj-=X&_6BAgT^9YskL1vG_9r9sXK)1a{XlZ&$b@)QVww1QF*F0tI zT1q`dOY~r7S8IenYp|&z`~v>?E$sNsZ1^#9oHrAe3}NduBN4tn)Qt1B*~*t#HyQ@= zD4hnc7Sn0r>*4Y4P~;(~nZwLt5Zhs9acPO&s)fj^`K!ID1hI{(6%p$ObRodPBg`VQ zFSenOQS2RSwpQEw6y)TQ%$7?C^W~AI&W$|8Pkd>I`ur(SQLgaPZX=%{z*?hOH(-E0 zMl-<2cECSHKsFCg;~xtnETrsM=v2~j)~UW8!zR|rYrI`wqn%%18PD*CzradW#WnV> zUT%%IcY47`y=Z|CxB3Y^JJt-PGy0mog0Ol>gzoNZ*`VxMx_=xC%m6?9zH4yXr_c%G zO(&VZh8Ncq_|zX|D{mpR!~|yL+%oY%`v-#lsI>oF- z{&dUEnQGQ25By|*%SU$IJTsiG-yczi#!odBsTjRJ%IreZrkf2&;kEWQi&<;6nHYm= zbo}XvczUe2SwO09$)BIW=6rShPn`>60*C&`Y$qO&iViV04#O^5zS8eH}#a%gv1?fXS zb?@h+uFZV2l62b=y)fUbOm6v!{^%23vcPP|hW{H2%)QbSDvdE4vH-Dv+(o{Z)xcH{ zseA(U5SSU%T1U^FFuNAzHOSc}(Ab+skI!Z6sJN?(E(t;D6p20VIG0WyIfb=X~ zve=B{y?O6qR_8m}h1-L$ubWO=VwNEj+@epGn8mqh>{3=}#)2psx<&(mE_un6B_`k= zsG+(RTEQ=ebMK@b^g|02hF63DMzXKO7N$!+HVcqn-B@=%Hefq9~FD0!8waALPew zvrgC|xp5<0+y-x6Lp$tjEvs%v7b1{tub7b>>airELuEa|E}3n`P9_ZOo`o?I>rwDBcpq zoVAxIaGHJhwU87-+rHvy*9|+&DDLj=9V}*qwtY=#*C!S-3%?hcm3EFSrCK!yW>sAm zddG|ov`rtgK%sgYpLu3f-#*S!>~fA>^{$XSPc@Ht$E-=GE@snt&(&OyKqE?RumrVU zjg4aKo!`x7NxE zcEpJ`L|hr3R8~`H>>DgIU$PLVykS=40p9TjGj20TBDXexTke5!Z#A_#omN#XNjJX9 z06qYKmHW*2cu%b*ErJ47hHvE%B^hJgIAEr7teArg ztD+sN+*7d5)KW{kt!s}F=^=JC!4MA|Vu)FG#8yv1>|IAS-H2l`V)?_?2wL|rLmX{K z9N|M`pI~mgofXDo{2%uSb=VeE_DSqQ;5Q(6dy4NG zsu^w{zQ9VV9XA~))1}jFVsYNC>zq#)u>;qC$1F)6xgj@fHVt~@9kZbKDxCyc1$byz z@cUB~urFm7lH%yOw^^;?Y37hi_8T^|-{3K$1`cDdedqc19_v%Z=!pG7z7+;S;0_P< zuaOUvdg|ZF_Zd_JJ5*gCl+?kBbq!yvrD|4RVKuP6cWr^MiAj?z4bvu|IyWd#of{7d zP(gTMvsmZ&G9H_lLJ%07Mu&W4I(Rlu{fHU7%5LxqpTT6==KPA?QG_Q{ydbOMy={j* z@D$i5Cs{My@FB8h8Z&CZ$Vx*84CoItEIR!YX6iLN<`o+zuA)0S>8d+vL$}aA=Yl64fXRXo}F>VnbYnmG&@&8o!j%nGq405m1@W_(n(HD9k(; zbbdZzbmqfC$3_j}Jhcd3f3Vd@FVPb)qF4$)wP zhMzn_?kO;Cz%h3}Y3c&YXBD;*`T0Y?lZKf;{SKaAWO%+vSAw_S?AlW0gW8j7kXu!h zJ>$w0xBSuF)#iGXn!9JrH82m_^bEf?vavC z@rn=JIf$L_11Q@tgNh-N3R{0wm4Q~|M`2%56O@eh8d>k?gA%d)e*$W*GU%;%G(k<8 zP)+|3#-nTGXZL+5%@t342xZ!0cy`mH;3*FiatD{3f&)muku{w-C5@A3PeD-*8x*`MD;V%D3}_W1T;u^w zZ)ebM+x{Udo&CDIQ-8&Mu@`ezZof9 zq-G@OP7%^bpjUvdLHwyCW`X#KEEZ?n7#Du41Z)uas}it7pwoS%GOIZA#tv~|mK4gq zAs)H&&qw=-{?qRE(FSnei)~b_?UIPa+9!!vtaoLRvie^LUD#%KSMcfSJFm1W=drG17$<9wv#{G5p!*SrOC=F;SRjk^=sG2UkKQK*u)GH)5esrk z7K_&)<1!y4oz7x>AYTb!*;h&dEb!Hm2y_#6$Rg!-ND1J>pC|!**!|Ff8Gm*ED+3lV zCw=LjHXJt+O@kKwo|20P!n3Ogv1?R!eQ$62j2S9-6dmiyB1i#>;g{<86e#m zA9@|_IO1Mmso;R9Qo-3G+P`v->z9q8`g=<0<@!>=1wlBI0>5kGLT-y#{*`;!DD2%3 z!~(ztH#jj-y5QF4ns_7g6U@StzZ8TY_E>7JO=@2T?oDa%-Poj2eM)Oa_= zg~Apwe8%nVG!Iljt%aVDqfpc$o<8HwV`UHtb8&rIzq7EGbXkiy?<^by%@OO)($wz8 zv+gFC;eC77J=sNjx-%!he(iM+ISM*KQ#}CzX{-0K3F6+ba%Z?8a{FYA6c7H*eVO#g znECKX^mk5rmI~%gN=WF+@EXtzpa&h+Kora*UZL3IEr*GUt zBwbst0$peNxW-{>?<*;F%Qx=H(vw$t#nQJrkHoZJc`5D8H||?I`hpqn9 zv%A49JJu`F_kt9gbN9F|`FH=aOCPKm9jT%X>(KS$=zq9}cBFGs6oZFu!~w;b!|+Br zAVtyN-z7fu5BH*IA>a!wfFJ|u7h|Yc2_g1^)Buon8HVtO?tG~ugpen~weSPUKOF!v zArb`%BHkISM4SK#A_^1uW3xa6Gzgsip=TbGVx-nUt+zqDz%ApUHKaSRT_8f;A{6od zWy&4zf&_O6mn#ubbVet5GJA4X@RpM>CbMg-q{0lrs@yc)A57~~i-2HENVTU*28QEr>Xc0^Q?fzis zN2taT_Ju)-a0&yN&0bF^F8l>$r?)ZW3&5!Gaf`U@J9oE|Pcb$ZA{#uR=DB`P6Zp2L zG0q+d`N3RN#j+Z`p{8H};x)OU0f;$0k#K!56kh{o??{Wd_dECI=(q5+T&C|qToaGk zDt&Q}knuQ~BfE5Q&?ROs_cTLVHiaTpg7IO;a$i%u>GQ5R!G;D;lP7?8kd|niuV!u# zJa@BVIG1~BYrTF&2VaGAjwclIhRt+jt_n8R$30Q2%D7f|t2|9%pU1Q$*>i%SI2S>R1_IcGN=c%c_RU;X zAB#?X1_ONt;9B8QGoTgEi8cBbxfN}EV5vltK1WAiUfio zbG9w?02dpA;h=eGqR4p1(vUBRbmCX(a$lg?17o5!@yooSYBjPNSR5rlqB@R-D4yt& zJ6`5uZ*8qvDKh7IiG~)4WL5b5exIkoOogM!TN6=N!`mcu`T{z=@PE(vexj3+ih%TRGmw{umiA3 z7`+EEARE1%DCgD(H#SD9{9s7DH83Zk^guA|Mf)avW5_{g+Kfg?$S4CH;9F@amc3M2 z9%yY?Pm`c7s<0a~SIlM;WFLnJ;i(oeBa{7PLTZ4l%w+#NNu5i+a0s(eMuEc^kO4VD z`75KqZy%>oAbJ#Ib)&#>j8jH|_rYP9wV@9_4R#<>B{trZ z*?nS)mT8}hJ+8xKCg!Dd#MxA5$sL=6vJR7Zl%5_Y^7yOmWsX8$jb;edLsE)48z9+D zkiJ_Y>RnqsAV|CiJL})YbU{b+zJCpLOhD;SA=gkpFKOkOz@( zP}UiQe^7cl1|gPJu~4dsDcOoV)#?#iZ#vsO3FR_ahP^G~9bH*Lii7LlyRt8J9-SMr zQE=&j0U4KGl)qvW`oKvinAb=9Vyw<8^vAfiTLliC+Ar~J@-_HiPGCA|be8(6*LlaQ z=hAw_hEs|4#Co{Di`VT+Z5cLMauz_lZQFc)<5(@b=t_p{S&Qeda>Y*i6W%#EGp^+8xUGAl`DMbM8^XQPUQ21ki^qt8@9c6lVw6!!Sl9l_fa z)jH-8#MRK`zm%H1Gd%3^qu15c5cK;S;1xy-zstPgX~|x0WwD_bE7FnRF)V~ag2yo+ z^^-PDj(JV(ED1)ci0V>Dh}q|mE1~$Ll8YL16k`37U=07798T7lbg8V8;O6>u!41|r zGUxk!8pcIw)*!DyX3g=e3r4D8<<6?Dvle)&fS0XHIx$#03xyBw#mUI$gKy`IM?#_QpV#0M-%KO5?&M7)-O5!d63|S?!@lb&X|TNb5Bg7BCas zWYf->S;=3ZRIO}euJSgcA1yJ?kpJ9d`8y#0O{Oa=Z74wYJWsO^<`T5B!ksz80*~V9 zn@u!IyrxBPtb$>Rw@Rx5!C4L`+;Hn+$Z%<$2oS)ZUSe?V*CJBrR$h? zsypUg+KwrnhmL70)U^XlF%t$Wd8dK_EeQK{=8J`GQK2ynn7_7VlA8wG9E1H<>HvI{ zdYpx2K|UtvXv{w;A02b$j>jF$+OR7HYrjy)Mk=UD$5btX@W(<)hsMV0a7?O#3{+IH zP`u^Ne zVHmSGTl%DAllkt7c4djREw( z4vl9LAEVp_Mret^W~-iQTZroUl%@$l*5U~41Dkg%h18N`!Yzb7t#LQzEQT+Nm>0lS zLR_-Q-%;#cZ{CpleJKtPK!F8~Bp1$liPOHsoXg4W>k_Dbt?U4D&FjjDH_3d(R*JI_ z?m%NJO)_1wmzftn;kQLCIFwmtQTR3(B-?UqVs7CT)!7#9FYoDcpNEFQ4r)<&2kZ!2 z^uYzR1CaeV*?%tmK|w$;^}dzxF3Fq}(g7PLx^YGAe2J~CJSPVt?>y)r#9849FWs zhf&sT7#*SXv^I>=-g-QNWiTv%9B2_g8q0d6h|7p^>@giKA7eJk2EnIPZ)un4G|D<$ zzM%BVEY%MWO^z^`q zrU_0jEQ0}@7o|0`?7GJA*c$xuo#|dvl3m7cCbIoHhzc-Ag+HSS0*OCkQPx2;p3>8U zDBUgnnOFt`L@%|7*G^`AQ|wp2pUfW7;j#j=QE<78>Mh|?fwB&lN=i=$7a@bY!kxQi zDJ*j>hm*P9&{oxGrr(~f#sU~Xd0FC3hx*#90+9y#FeMypHX%>{WJi1Nn9APJfwU2` zRX}Pc5G0VcpsWLFE2XCg(&fBqV8<4jnA_ zVKxdZJE+ys)3ew<9Weimc`3krLKT;Q`3z+pFrQO;dcZ8>0pnUw=MB|)0-BwB zy}kJwb2q?putjVrWIauF+w--DEwi3cA>zV-JbvfE|9oTJR&}7L^g!TY&~uCU z(2I;swPHpO%-aCLt5VPDA@1p9@3F|(Z94b@(5b57TNf}91zSwp&-zGXKRR&RTsG8} zC8Hyrn!pA}Gkr}}@LiKJSPqx+3I)7NHVz}QJT-V?0CJdsK{_5JOd_!+=!*+e=&7n; zaGm@%+BAxgx5Wd{;b&k(HjEpMo;v)f!Ym4}^EPh`hVVlj(L##w2dh0{I)^kD!%p~= zhYxN~`Fu*&5}X!l4s)#ZdF7Q@^+s0|Cdx_oe_cf(FC4ChQ*Wlz#N;{({oYM*=z$JC zNmI?3ZE*okJk{laH@k{pu_xfEPIB(SQ4VJ#Oy3={7lXC6UYIM1RKwXQvx&cBo^KN! z#ju#W8$j08BvC{Q9Z~08c;!yDK8K(fFoBc4>Qz?#?D2BqjQ+IYA*mDP;+7B|;(+s2 zKsep`su93(vqK#4yzUH#YY{u3Gq;ao-iolHiVy}v>&AdFo_oqp!6q{aP}@_fKMRcR8_bXrI%1D zMhT}OLU4Dx+3;TjP`nc((Zv)FpoGqp@Lwn$MCk{Va2OQ^0y+ZP0^xp?UTYE8E@eGz z$6CbZr7Yid0#dfd0VVONrECpGyUJjYgz)$>_R3g#i~V>lzUK{}Fc`E4I&>F!f!ps3 z)`dKc_05-i{1JSWJAd&CmW^*J6^h@jV7*?Uxl!D=oOO*7DOerkN2JI@iAXUFB_hRml!z2FQ6f?-L_TI@ke{Nbw7dBvL$#5|QF*l!z4jQ6f^jiV~6H7)r#6 z|3ry6@exYIiPI<%C;o;KapK=75htAWAYoT25ITbxSrZRqLthH$N-yx^{}`dR9|lqR zi{W4YBwD$5v;gAdh8qJSH2?V4m`NZCQ(D8*F|2Fq`g=^;S=*3Vu=c$e9)}D{;o=d> zC=k`FyMW9Q%J;xm9&lA2X-nkU15n8p1tlWSVU&nGZ=powc@HHb&!15u^86JgBG2DZ zBJ%uz5|PIR{~$vmPdAi^JbizGUzs5G48cgUC}U6}@=QaC$TJ@$BF|Ekh&+`j5qYXn zBJwn#MC93s5|QURl!!b(M~TRD7fM8)+y_xa_6Se@LcDe*o0pr9jZEcx;7Rd!E7|ZY zdi(eJ7SUG8#*Wwo$uNAucYtBJU;Y@3fJBKYC#svO1iT5p79*~$WO?ozA@DQY`-p-- zM;tu#RYFvEPyQ?3Sa4NyV-xVB!#@>*15l&%_7>|_;)?$TEP zO5i}}uOQx3+nY zYgqSluEr-3GR^jc>x_7Jw}vb#vKhtqt`O`Ia4RRmJOZ#T^bkxBW8#@ z7r=i#JhPUKl2|jtnl*;(2(0A<*3AlQoUo!gcD#p2n+pL+pS@PS$(lAPKw`*F#UBL{ zp(~V0!=HT#{?JfmL;qFJA4>W0_|xcR^QTWu11#~ZmsLr$skEkzAr}>G4rysq#iPxI zKw{+uLBf!WiY*Ef!csk34kp+_?#vJ#TPWqnV@p{bn>>4C8W=&>nk26EvmWBfIyP9M z%_M8u7_w8*<_;}ws(G}z5Rlw;VIeVOr{a$Si4fECXLW)<c=p*;L6W9nQxg79A8~h(b(YvP z#hOiqJXLJEPRl0vAeGT~z7V){dNwU5C?#ABc@i#Bat;+-1Sb?s;|Vn;b)XSEaOqFMCV#9jY zGwR3s&_o=ipTWmz!DEMo9ku@tcVHs^FB1YQ-@rd%P&TH^Q`$YPBwJWfJ{JD@sF7qc zWIqAjNoMWhhDKJB57+Qe=;XAF0QS$?4(D^CkW_Hs{Rl}KL9v3VQhWt=MLvj?~~O$QE}*S*Wzo(;{Pa$ZF0R3)d!gS@dD7w$?PXE@%nd zRMQT%9cw|B2PHmt;0{bwEKh!Mo{>Se5oFLb{Z$NJ$NGqQ%`7KMIlKtvHp?N_*%Mmn z+c0(P_|anuMo$_udK`FuvZd>6$WCkNK2LD9WaSDjSBJ|h%Ep#CuMKm|vSGNXKX$_i z`=LzWLt8e?sUNvvghP;lrZ07q`_ejjn{Q( zLJt`!IRnNXY1;6Y_9x{XpEFY~zI+YqB@Goxdft-d8Huh}Q9ft**s(k(Bc*2SeNozR zFo43dtWrC2Q!jQqPCKvdC@@f`>8WC3)T7O`7|#Ww6n;k?YS{e2?;LQu=bvw2!;RhZ zz_UMPWf{hJ@tT|1AWrmx>)HS1M89$~TXN2$?b|ikE?m~p_De#CrkPl z2nqfSSFe>Ofw84CmTBb%QWDF|Lw<0CHpjAr(DMYgoKCPS@IwiXj&cDuNA)q8Z{INYIE*HxI(sCG=A`N@*jF67Pq}OA;M4 z6&HIv&PH-pZ?1cU6)iO^=wh=kF35xTz!E0U416B;>`_)*Hjab6eNlimGg4agdFTmN z;ig${slKsupJXedybK+ti-;2uZ(6TK$#gLo6BCo3A-a4)L+d6$dxf7fpA}C($wo?x z!7Bg-zH)JG3mYU&Li-r?syCdMyxh7NTvAeGvUHUaSGsf8uP!)O8ZI*t5SxeCX+OT0qWH^}-09<-H&s|LP{|py8`?I&U-K}~qBMi1(z6$}0Ar}rP!oTz+dM!aBdKc~`9*HQ$Kq5X~ z^v@Pm=1q0nXx2V<#W|a-WBXVrI*12g;;kTVs;LX}Cf1N#7?k*42Hb&(iVMR#VWRS3 zR4xpiF!_GJa>8UPl-n!^)d`cy1@QLQ__3`EfXVi`ZLLPNyrP`v+AM@Bq|aA94*=pR>^Vw=+IOl@Yt#;Q}m$ail*+~ZvRM&hpKX&s8b07zew-J}Tjg`=R^de_F zp1VOQX8qB6J54-wfc2L)`Iy1Rrn<_!oeq{GIIDO7P9<3 zDtzmFjqu?aPjxT=@3B@z!og4ze=JA?OPKF5DJSuZ1IpIvc}Det3xcX47eci&O&b+d zg>C!9uU}&GqxiXjQaB788USy3)0?R;Kq2#mbtOPrB{2d4b>g8?RpN07s1rX{C;pEv zac8;e$zA2}LbdwQ+{;y|3g_R)vCl-AfQgXz6KM;O}}EJq7rK3ysF!0KtO%_Kh=r0T5VzhgcEpA)+OGqPP|8# zcmh9U5OvAz_&$WxnLc<6q$-I|LO`83@m5vhYzU|mE7ghBy2KrE)sv6Mb?x}JI`xkb zQ0qObPW)Dv_{d!G!bphR!|>Eg`pkyti-?=4*Iga5#d-v{Uk4vo08 zSI6FBV;P$yw!Fe7Et(G)-41C;>us@IZv%WV3|@;bXoOeeZ-@(vHOU>p+EjZR!x4{P zxnZ7igLvsd*3Y#t5nOVRl@2XYf^gjy>i{4Bxk`G=sA&VKU!tkGS>-8;{X+Uz$YDriBf#SCOoc?_xUSTt24OVq+A0TOj z2qzct{x?{@q;lKs?tbE@Z?L{naBtk*L+p2yQ2`DfWBtU+qpTB_LO99>;`R2U@P!w+ zK6jK&!0YEn*=W4>ImY@AA@FhrhP<`#f7yVz@F_E;6cGVQAm{Hn#s*0lF5BzwB|dyC zk>Mh#5<>@y%l?=#RGfUAG1OjmoQ;tlc=UHya*k$lLsmRleh^S zli3L0ldTFIlhg^>laC7=lgSFklaUM@ldB8nlhh3yv#ks#0h7HB9Fy=3&6A)H9Fy4& z$CIWI9h1Ee>65Dw9FxKj=#v2x9ka&~I{}mW6C9Hs6Wo)_6daQl6t$DL6&{o66wZ^Q z79Eq+70;917aX&{79IfsHj|+b8-w{5hxr!)hxr!*xA_+XU+Mxpyti5c1m(Gx@VW#A Z0XCN*xda{o`nMsx1jPXXLYL>g1akQPTMYmJ delta 209 zcmV;?051QW{Sw*y60k)90XDNn0Y?M@H;gfg5eWJ`t*Ck2Ztp_t#ODkP;&pCH~E<6{kd>#+yocH^A z-|sv3&b{|_yKM8bGT9p0(<wCEzo`f>j=~`XXeQVMe-gUJeZxVozsFogxTO?hL0s3;-FXvb}jq4VmlN zd}DY#nG4B#R40%@$oyu+ab%!r)wnD2Loncvf)botqS98tYxtcfmNv~b+a6qMHCNcQ z)ytMyYU{LhmWM30=1Oh7rLkUHS5sM0Ur)CmSz1wVS^Yqz$9xIJmw539v~>+jty*)P z*0f;W9a)*$5_9eH^6HhgEUnpAuD$)PlDS!B<~mEcwz^h`sx4ckt*`c9Zm~I&o7Vw6 zh+kwQBPz!ThlaI;=s99oCT6zzSPsh0Wr?4v#OF zC-?JXXiRI|SOOi*4k4AA%ctfER=htYHebgL5f-fhG`YiKolH`sk^kw79SznjZaTG8 za|ZIKq0DTZNU2U*>}d6`D5s5z!cog7G`XEygK90y>Q~iR9Ia5}XqA(drID1MM0VI2 zxYAr}bJ)3Ins98MH19u&i8axhZPk<{-9ZS%3yl36%B(T1b4Ru6XZMP<3ZHIy7~L<& zD<03pDn}4*Z;8Q`Rgw7M`q`*zvd@*FO>+FvU^;%gEj98f%0yt6(?FPQ zBu)!suyWJP$Yu&@O2{_F)!cdks^4UWX8hTvi>ODAyHj-dVtX7Maw<1p1k^`&vK5+= zTeiBL4&eg)BKpchMFoR+lH(3C);RLf0F5{$^j42?KOS(rh!hHs9Qkb?Bv^rapI!*T zxM!z6)VnLg@Jt47>IebqnToqQnvve?9XiM3orUZ8QpjS?-pwh**}xoA@O{s$M~i%p zw|g8*vG&=8P|6&=oAYq@vmVFLE$!$LuOsf;5&|pn@h$g|kuojv1gfV@<=$n#oZlP` zNjUL|$Q!-t4|xv-*FvVAtHBybF9oTsHfQA4P6RLGo^2V)+lULdoenRMfgUQ3d-rtSO~yT)H|3;;kt->|N-lS)HcB+R&8g|yqJqU(|H*Yk{$U9dc^_W; zNgS}&j@HO51dVqyD^G9n+s0_OM3D`V^u^n!U;_~|8=9T6cM1V@gn3#G+q@#*>Qz7o z{-Ada8TWMR@gwibbDDp-&^O*~RKi==7rJ*h6Z+=ZLYajibpHsU@}u5fifo$k!lQfP zE7nWHr?EMwe=&Ew`qTg6=Ze!&h+iB*{I1W9x41hbviTK6`#%2% zUeljBaop?WR1!YiuaQ1Pr;?EHGoj`Z@W=ge;5kGO9XHr*Y@d56CZ^Juvm1iHBi4bfz0A=|(m_{7o1dR^mOi2B-4uen3g#c>cW!_|&&) zC_S8?H4$?0%yV+G?JPdGo59_kQ}LAZa+rwMUdaec{fqDF498zOlAP`50}(ROFo}5I zcekNC-C?GFF9(zJrti-JMQN1#FUesJjiS(rqVVm}kW3y=BEn3f;KTUjr4LYpK>FAZ zUef9x?*wEQNN@bfh}wm0icDWbyAxfW#f(T&G0;MC=b?21?GsntMB=7USpf3|qZc$m{5&O*WOV&b3AHQ`N!+0#c@6Hoq8LJ6*D@7! z09qmBjE{ub&?w58@5{NuwJj1>GgB(3B$|Y^xxW3GK_ zC}%d5NO2+|@~NB1Y#-kvtgf!^L`VbV7r`2JNdrYFD1u%?Gs&K7?L;7N?F6KO4rocB zVHSx0uaEu@>kc8c?GUt2vzr zY;w~p_SrV_;ZxuzYDFXFXOM^uEFxc%!IaXbmGU1E0GvGOOO)$e20Vt)%furkn}0qV z!U(V{8+M>S(=8qU&<&)A2V9$P09hb9%*>P;Q>icK0siPT=s<&lxg!tarwv4Set%oz z>uRhOw#KE^w))zNvh1>oGV;YD+g4p}sdKa{$g;Ac`oUUrjdc~?s*T4-WO126<<(1V z<|>PWBlk{r1HM4xsX0$r;my^wh;VRSoeoD3=v^P*2&ety^Kyjm3o)$$`Wa*)d@UfZ z)qGz8hs3fLtqsRCog)jva z{BR-Yn!XArG0t(9>92_r$nahW8G7k(kiLjz(Y<8ySjo+!v&ZRd1)Y6KXPC~uqrUs- zY=8vg2I*{%iVP7%Fy|s_xm-Hyr?Z80=1MX_(4nl7^^ECdo-1^*(%13eWN3XQl95AO3?BZt&p{!2>P?iwMdeGE!bSDVmB4dE_4}<9E~?)gDO^;)JEd?@{qC2-MfI~t;iCFgOW~sW xt&+k;^;;)}i|Y536fUYCFM;QZ>bJv#2MXuY_5RIJ1%UN~)(?T)JplTZ0l!E2xPlRjD@NXhiM3_ss@JQFH1a&%6A--}n1{ z-<$j1qjHDrpjjq+M^@v78fEF=;hvwHVEH+`=|Rd;zz<*X^v2)I&a>r2TrULR8=g1V zWE7wL$Ln8gGMZ01y}x9WBtE$&f5;}uY+?b0h7+goS-jE>);3+be^$F*%pCndG|e1UX4rDd;gc(KQDaC5`!HVnj9*Xu2f@2vxv zg>SDPhqKM)C{cln_l?BLP0P?^KC~R%Gyovm@`ugSWnjdb%5?1#g`1l($QWx3o{>;$ z3=kLN6_r`qg>03tu&n@b5nUaGZ&xmYg?Lug52#jwQ*{xz5D!8`fsbeVT6W@gK)+I8 z!y@XfM$VQq42k$wO(8la$=JG04o7h9wwdS%4VfRWn6!&04iK9uASVjXB68HEz#;R( z@EWofUFTtn$#zt`0>8373(@~ruw(h3JlHw6F$~}FKJ>%KKS}I2#ZKUfZ|=xMo=O&v zSJcXpyOQN~0-Erl+5-^BqVSfTa)`z|c4nbyDIjE*97RY0Q|*jRID6M;Fq@~DW?xLl znY-UW=~C3?-EuU8MMcZ)`%O5s-WSL2iB2G_%7I4r(i}0H4kOo9uc?kO%8Z^yuLadl zT3Tw^yr&+ax0P6^j2tGG(IBy$T~^Ucsd0qp(#^~k+PAk@>VZgHgFpT_*#BJ{%GO*YI`mqh+&}>_; z-?y2r^O7&yFaIf)f4Tp4FLZ%h4)4>lzC;Ud)bx2*(}6urOBl)axn8y})Ft^!wp~$# zw(B7vyzW8_PCMj>9#b>fzPRjA0}@nH9VRuT!601OFb)KKu3-*cr+(w`+(u=>W919C zm^9W3Xtav8_!)kyvbo=pDwihr_RBGn-*_9&-ezS>Qh%FY7t~`V$&i6B7F+%b2JU*v z2|u)uTB5)5suMAQifp!7`^%{9MCJYsREU(M2v0cm4u!A^jMckrEL6R4C3L6D)u`rq zD)E~-UWux{D#0<5$Su7pvA)$&2^Z8tl~jq2);(~HR$^kp(HHiByDoI2^Uq`9?+>_z zbA4Ih7|FsHy)2w>0N-u49r#S=!s{6Vj zms&o*5(MxvzS#bH=u^&EJ!cCLlhwGm(ho1`7;FDM^+Rd8%5*#odY=;pU-wj`!+tKecs<*^`K9uoBs~!HnmMxw2fF7u^wkr_NyAzI(1|QFh!3}qO=&I$!ojSft z&s?>BxRc?Y8n@qz!e9MYfqqca7pox_`S+d(!m!Yli=yllUijBt3J9|t?z#mijIPd) zQ|Tj|EFt*Hz0r(7w_pe!yGM)D?knxm3+^9AGc}U*C})Ty-GYD21jwP;g0Opc63TaG zEb3N3j^(ZHR6sdABfj-e3E4DLv@*uYGvA_n^c}(z=la4XMnkdyl2E0CM;YtHZnB;Q zIE}P5l@r|fKlU;eAa3IHQ3C!7X!UUE!+;SCCCu^#jWHG&&l;KhEBg*qJqRinRg^z(+sTBLx_0Nvn{RC<@a z>r5G`P*J`Pta~)ndPD_HfCPbcoNU!l*M5Sn;A_f&ilLjA0hATc$14u7r!2Mw&%5Y{-44^o@)o%c-;TV2T zjB$FIBGEk@2ny7}ikLy>`GL}2rqzD%HJ~zq-xy@`OBB_j;ZIX?;3cRE0qwShv<1+R3W3ylLy|Qp5J~|xN=h4o;8pZnhf?K4r>I72Y%t_GeXSgpe#4G?H0t|?BgM-tbva64C+b!Rwu14K$( z$0J}I#51mObSX6w?xIK=%QPp;a8euvqa}A?)-zGiA0UGfEP*#99jjxgjtsIf2JWK_ z8$p_rU?wAoCf|*aWU!u2jit9vg*7V{DiOTRSYk-J3S_Do&M?{~_QFV#b?^afW0*+N zsRMt&)_w^f^Mrjaahc9Itd-&n$?z^5k>XS-^oD5G(sOSRX^et^wT5}q4TVc58V&h{ zA+z(!O(CzXD9AUN@=6v&7)_=XE62pe6;Qxp28b=mGvu0#dAUW4SH|WS#pM;!JyYIF zQ{3{R+|q))m2oD@kiF8z5Nk9oFNln-W*=J-#x?e@uL?hxjLEJkInp?=2cXCPd)3kx z9Q_30cB%qpYfDS=t1Bk`U$j=!Cy;TqF<3vxS;W6-?JqG@7{pdPB}hB?+nJJ{8(z-FmE*3&6)56grXe(PLKhp)Vo>$=>lW zIK=2fMbUSO(wquySy4&u;-b<*(}?Byg{8C`)nYzr9}h#(n^HtjDujZOj7x=4{!3}# z6GK*!4^truyhw8@M65lgq5PfVKiG3zP5=HOo|4l`3%gXaui+7Ft($&%iL=;f9(`*U zi`nRNHo|OlhQ%Icqh?AbUS%V*KrdQEFI#C<#gS~(#70?c^oWhh=%`d%Aulv6&oixB zLEX@YjOay*CqR&%$ue_QX0B)>S7YW%%v^<;D=_oyW}ey1vzmEEGtXwGnaITnkU5u& zXMZD*#IsLHNj&?MlEkx@FiCtXGsMNWFhg8?1vA9OW4m~i`~}aGvsQ?0XUb@83G;xYaf^K83G-*Rv-eW0RcLf{viT#xJFpk delta 214 zcmV;{04e{o#R%}k2(W7b0XVa40doWbGdYvt1?rRF1ss#-2F;Vp1{{;n2g#GP2ON{N z2(*)#2pp4z3Eh)v2^^Dt3d@sX3LKMV3;UBv3mlVt4D^#=3>>qJ4IlxNa19-k$qvnv zs}3ELvk%OZmk%7Xn-CcRlYbB#lh6^wle7^Wlduxslavx1ldBWwlaCV|v#t~{0h5vx z9FzJL)sx;89Fyx7)sxQ_9E02!x7-&2wCE4mApigX0001elqhH$Y9^O4{Q)tx&;o`Z Qx0xUUrvU*umq8)|a#5aB!T
-

4. The Evals Gap

+

4. The Evals Gap

Evals are surprisingly often all you need.

—Greg Brockman, OpenAI’s President

@@ -201,45 +201,45 @@

Contents
-

4.1. Non-Deterministic Machines

+

4.1. Non-Deterministic Machines

One of the most fundamental challenges when building products with Large Language Models (LLMs) is their non-deterministic nature. Unlike traditional software systems where the same input reliably produces the same output, LLMs can generate different responses each time they’re queried - even with identical prompts and input data. This characteristic is both a strength and a significant engineering challenge.

When you ask ChatGPT or any other LLM the same question multiple times, you’ll likely get different responses. This isn’t a bug - it’s a fundamental feature of how these models work. The “temperature” parameter, which controls the randomness of outputs, allows models to be creative and generate diverse responses. However, this same feature makes it incredibly difficult to build reliable, testable systems.

Consider a financial services company using LLMs to generate investment advice. The non-deterministic nature of these models means that:

@@ -250,7 +250,7 @@

-

4.1.1. Temperature and Sampling

+

4.1.1. Temperature and Sampling

The primary source of non-determinism in LLMs comes from their sampling strategies. During text generation, the model:

  1. Calculates probability distributions for each next token

    2. @@ -259,7 +259,7 @@

    -

    4.1.2. The Temperature Spectrum

    +

    4.1.2. The Temperature Spectrum

    • Temperature = 0: Most deterministic, but potentially repetitive

    • Temperature = 1: Balanced creativity and coherence

      • @@ -358,7 +358,7 @@

      -

      4.2. Emerging Properties

      +

      4.2. Emerging Properties

      Beyond their non-deterministic nature, LLMs present another fascinating challenge: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren’t explicitly programmed but rather emerge “naturally” as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against clear specifications.

      Emerging Properties @@ -370,7 +370,7 @@

      -

      4.3. Problem Statement

      +

      4.3. Problem Statement

      Consider a practical example that illustrates these challenges: building a Math AI tutoring system for children powered by an LLM. In traditional software development, you would define specific features (like presenting math problems or checking answers) and write tests to verify each function. But with LLMs, you’re not just testing predefined features - you’re trying to evaluate emergent capabilities like adapting explanations to a child’s level, maintaining engagement through conversational learning, and providing age-appropriate encouragement and emotional support.

      This fundamental difference raises critical questions about evaluation:

        @@ -420,7 +420,7 @@

        -

        4.4. Evals Design

        +

        4.4. Evals Design

        First, it’s important to make a distinction between evaluating an LLM versus evaluating an LLM-based application. While the latter offers foundation capabilities and are typically general-purpose, the former is more specific and tailored to a particular use case. Here, we define an LLM-based application as a system that uses one or more LLMs to perform a specific task. More specifically, an LLM-based application is the combination of one or more LLM models, their associated prompts and parameters to solve a particular business problem.

        That differentiation is important because it changes the scope of evaluation. LLMs are usually evaluated based on their capabilities, which include things like language understanding, reasoning and knowledge. LLM-based applications, instead, should be evaluated based on their end-to-end functionality, performance, and how well they meet business requirements. That distinction has key implications for the design of evaluation systems:

          @@ -507,7 +507,7 @@

          -

          4.4.1. Conceptual Overview

          +

          4.4.1. Conceptual Overview

          Fig. 4.2 demonstrates a conceptual design of key components of LLM Application evaluation.

          Conceptual Overview @@ -588,7 +588,7 @@

          -

          4.4.2. Design Considerations

          +

          4.4.2. Design Considerations

          The design of an LLM application evaluation system depends heavily on the specific use case and business requirements. Here we list important questions for planning an LLM application evaluation system pertaining to each of the key components previously discussed:

          1. Examples (Input Dataset):

            @@ -673,7 +673,7 @@

            -

            4.5. Metrics

            +

            4.5. Metrics

            The choice of metric depends on the specific task and desired evaluation criteria. However, one can categorize metrics into two broad categories: intrinsic and extrinsic:

            • Intrinsic metrics focus on the model’s performance on its primary training objective, which is typically to predict the next token in a sequence. Perplexity is a common intrinsic metric that measures how well the model predicts a given sample of text.

              • @@ -983,9 +983,9 @@

              4.6. Evaluators

              +

              4.6. Evaluators

              -

              4.6.1. Model-Based Evaluation

              +

              4.6.1. Model-Based Evaluation

              Traditional metrics like BLEU or ROUGE often fall short in capturing the nuanced, contextual, and creative outputs of LLMs. As an alternative we can consider a “Model-based evaluation” approach. A common approach is to use an LLM as a judge. This is an approach that leverages language models themselves to assess the quality of outputs from other language models. This method involves using a model (often a more capable one) to act as an automated judge, evaluating aspects like accuracy, coherence, and relevance of generated content. Unlike traditional metrics that rely on exact matching or statistical measures, model-based evaluation can capture nuanced aspects of language and provide more contextual assessment.

              As discussed in the paper [Li et al., 2024], LLM-based evaluation approaches generally fall into two main categories:

                @@ -1225,11 +1225,11 @@

                -

                4.6.2. Human-Based Evaluation

                +

                4.6.2. Human-Based Evaluation

                Human assessors can judge aspects like fluency, coherence, and factual accuracy, providing a more comprehensive evaluation. However, human evaluation can be subjective and resource-intensive.

              -

              4.6.3. Evaluating Evaluators

              +

              4.6.3. Evaluating Evaluators

              We have discussed how LLMs can be used to evaluate LLM-based aplications. However, how can we evaluate the performance of LLMs that evaluate other LLMs? This is the question that meta evaluation aims to answer. Clearly, the discussion can become quite meta as we need to evaluate the performance of the evaluator to evaluate the performance of the evaluated model. However, one can make a case for two general options:

              1. Use a gold-standard dataset that is used to evaluate the performance of LLM evaluators using a “metrics-based” approach.

                2. @@ -1273,7 +1273,7 @@

                -

                4.7. Benchmarks and Leaderboards

                +

                4.7. Benchmarks and Leaderboards

                Benchmarks act as standardized tests for LLMs, evaluating their performance across a spectrum of tasks. These tasks simulate real-world applications such as answering questions, generating coherent text, solving mathematical problems, or even writing computer code. They also assess more abstract qualities like fairness, robustness, and cultural understanding.

                Benchmarks can be thought as comprehensive “exams” that probe different “subjects” in order to certify an LLM. They help researchers and developers compare models systematically, in a way LLM performance is comparable while enabling the identification of emergent behaviors or capabilities as models evolve in scale and sophistication.

                The history of LLM benchmarks reflects the evolving priorities of artificial intelligence research, starting with foundational tasks and moving toward complex, real-world challenges. It began in 2018 with the introduction of GLUE (General Language Understanding Evaluation), which set a new standard for evaluating natural language understanding. GLUE measured performance on tasks like sentiment analysis and textual entailment, providing a baseline for assessing the fundamental capabilities of language models. A year later, SuperGLUE (2019) expanded on this foundation by introducing more nuanced tasks that tested reasoning and language comprehension at a deeper level, challenging the limits of models like BERT and its successors.

                @@ -1317,9 +1317,9 @@

                -

                4.8. Tools

                +

                4.8. Tools

                -

                4.8.1. LightEval

                +

                4.8.1. LightEval

                LightEval [Fourrier et al., 2023] is a lightweight framework for evaluation of LLMs across a variety of standard and bespoke metrics and tasks across multiple inference backends via Python SDK and CLI.

                As a motivating example, consider a scenario where financial data has been extracted from SEC financial filings and require econometric analysis. Tasks like estimating autoregressive models for time series forecasting or conducting hypothesis tests on market efficiency are common in financial analysis. Let’s evaluate how well different models perform on this type of task.

                First, we need to select a benchmark to assess LLMs capabilities in this domain. MMLU has a sub-benchmark called Econometrics we can use for this task. Table 4.4 shows a sample of the benchmark dataset from MMLU Econometrics. It consists of multiple-choice questions from econometrics and expected answers.

                @@ -1508,15 +1508,15 @@

                [Hugging Face, 2024]. Its integration with the Hugging Face ecosystem and modular architecture make it particularly powerful for evaluating open source models. For further details, visit the official repository [Fourrier et al., 2023].

                -

                4.8.2. LangChain

                +

                4.8.2. LangChain

                -

                4.8.3. PromptFoo

                +

                4.8.3. PromptFoo

                PromptFoo [PromptFoo, 2024] is a framework for evaluating the quality of prompts for LLMs.

              -

              4.9. References

              +

              4.9. References

              [ALB+24] diff --git a/tamingllms/_build/html/notebooks/output_size_limit.html b/tamingllms/_build/html/notebooks/output_size_limit.html index e2fa984..0bc0f19 100644 --- a/tamingllms/_build/html/notebooks/output_size_limit.html +++ b/tamingllms/_build/html/notebooks/output_size_limit.html @@ -194,7 +194,7 @@
              -

              2. Output Size Limitations

              +

              2. Output Size Limitations

              Only those who will risk going too far can possibly find out how far one can go.

              —T.S. Eliot

              @@ -202,34 +202,34 @@

              Contents
              -

              2.1. What are Token Limits?

              +

              2.1. What are Token Limits?

              Tokens are the basic units that LLMs process text with. A token can be as short as a single character or as long as a complete word. In English, a general rule of thumb is that 1 token ≈ 4 characters or ¾ of a word.

              The max_output_tokens is parameter often available in modern LLMs that determines the maximum length of text that an LLM can generate in a single response. Table 2.1 shows the max_output_tokens for several key models, which typically range between 4096 and 16384 tokens. Contrary to what one might expect, the model does not “summarizes the answer” such that it does not surpass max_output_tokens limit. Instead, it will stop once it reaches this limit, even mid-sentence, i.e. the response may be truncated.

              @@ -289,7 +289,7 @@

              -

              2.2. Problem Statement

              +

              2.2. Problem Statement

              The max_output_tokens limit in LLMs poses a significant challenge for users who need to generate long outputs, as it may result in truncated content and/or incomplete information.

              1. Truncated Content: Users aiming to generate extensive content, such as detailed reports or comprehensive articles, may find their outputs abruptly cut off due to the max_output_tokens limit. This truncation can result in incomplete information and disrupt the flow of the content.

                2. @@ -298,7 +298,7 @@

                -

                2.3. Content Chunking with Contextual Linking

                +

                2.3. Content Chunking with Contextual Linking

                Content chunking with contextual linking is a technique used to manage the max_output_tokens limitation by breaking down long-form content into smaller, manageable chunks. This approach allows the LLM to focus on smaller sections of the input, enabling it to generate more complete and detailed responses for each chunk while maintaining coherence and context across the entire output.

                1. Chunking the Content: The input content is split into smaller chunks. This allows the LLM to process each chunk individually, focusing on generating a complete and detailed response for that specific section of the input.

                  2. @@ -309,7 +309,7 @@

                  max_output_tokens limitation and generate coherent long-form content without truncation.

                  Let’s examine an example implementation of this technique.

                  -

                  2.3.1. Generating long-form content

                  +

                  2.3.1. Generating long-form content

                  • Goal: Generate a long-form report analyzing a company’s financial statement.

                  • Input: A company’s 10K SEC filing.

                    • @@ -322,7 +322,7 @@

                    Fig. 2.1 illustrates the process we will follow for handling long-form content generation with Large Language Models through “Content Chunking with Contextual Linking.” It shows how input content is first split into manageable chunks using a chunking function (e.g. CharacterTextSplitter with tiktoken tokenizer), then each chunk is processed sequentially while maintaining context from previous chunks. For each chunk, the system updates the context, generates a dynamic prompt with specific parameters, makes a call to the LLM chain, and stores the response. After all chunks are processed, the individual responses are combined with newlines to create the final report, effectively working around the token limit constraints of LLMs while maintaining coherence across the generated content.

                    -

                    2.3.1.1. Step 1: Chunking the Content

                    +

                    2.3.1.1. Step 1: Chunking the Content

                    There are different methods for chunking, and each of them might be appropriate for different situations. However, we can broadly group chunking strategies in two types:

                    • Fixed-size Chunking: This is the most common and straightforward approach to chunking. We simply decide the number of tokens in our chunk and, optionally, whether there should be any overlap between them. In general, we will want to keep some overlap between chunks to make sure that the semantic context doesn’t get lost between chunks. Fixed-sized chunking may be a reasonable path in many common cases. Compared to other forms of chunking, fixed-sized chunking is computationally cheap and simple to use since it doesn’t require the use of any specialied techniques or libraries.

                      • @@ -334,7 +334,7 @@

                      langchain for a content-aware sentence-splitting strategy for chunking. We will use the CharacterTextSplitter with tiktoken as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model.

                      +

                      Here, we will utilize langchain for a content-aware sentence-splitting strategy for chunking. Langchain offers several text splitters [LangChain, 2024] such as JSON-, Markdown- and HTML-based or split by token. We will use the CharacterTextSplitter with tiktoken as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model.

                      def get_chunks(text: str, chunk_size: int, chunk_overlap: int) -> list:
@@ -359,7 +359,7 @@ 
                      
-
                      2.3.1.2. Step 2: Writing the Base Prompt Template
                      
+
                      2.3.1.2. Step 2: Writing the Base Prompt Template
                      
 
                      We will write a base prompt template which will serve as a foundational structure for all chunks, ensuring consistency in the instructions and context provided to the language model. The template includes the following parameters:
                      
 
                        
 
                      • role: Defines the role or persona the model should assume.
                        • 
@@ -426,7 +426,7 @@ 
                        
-
                        2.3.1.3. Step 3: Constructing Dynamic Prompt Parameters
                        
+
                        2.3.1.3. Step 3: Constructing Dynamic Prompt Parameters
                        
 
                        Now, we will write a function (get_dynamic_prompt_template) that constructs prompt parameters dynamically for each chunk.
                        
 
                        
 
                        
@@ -479,7 +479,7 @@ 
                        
-
                        2.3.1.4. Step 4: Generating the Report
                        
+
                        2.3.1.4. Step 4: Generating the Report
                        
 
                        Finally, we will write a function that generates the actual report by calling the LLMChain with the dynamically updated prompt parameters for each chunk and concatenating the results at the end.
                        
 
                        
 
                        
@@ -538,7 +538,7 @@ 
                        
-
                        2.3.1.5. Example Usage
                        
+
                        2.3.1.5. Example Usage
                        
 
                        
 
                        
 
                        # Load the text from sample 10K SEC filing
@@ -606,7 +606,7 @@ 
                        
-
                        2.3.2. Discussion
                        
+
                        2.3.2. Discussion
                        
 
                        Results from the generated report present a few interesting aspects:
                        
 
                          
 
                        • Coherence: The generated report demonstrates a high level of coherence. The sections are logically structured, and the flow of information is smooth. Each part of the report builds upon the previous sections, providing a comprehensive analysis of Apple Inc.’s financial performance and key risk factors. The use of headings and subheadings helps in maintaining clarity and organization throughout the document.
                          • 
@@ -620,7 +620,7 @@ 
                          
-
                          2.4. Implications
                          
+
                          2.4. Implications
                          
 
                          Implementing context chunking with contextual linking is a practical solution to manage the output size limitations of LLMs. However, this approach comes with its own set of implications that developers must consider.
                          
 
                            
 
                          1. Increased Development Complexity: Implementing strategies to overcome the maximum output token length introduces additional layers of complexity to the application design. It necessitates meticulous management of context across multiple outputs to maintain coherence. Ensuring that each chunk retains the necessary context for the conversation or document can be challenging and often requires advanced logic to handle transitions seamlessly.
                            2. 
@@ -630,7 +630,7 @@ 
                            
-
                            2.5. Future Considerations
                            
+
                            2.5. Future Considerations
                            
 
                            As models evolve, we can expect several advancements that will significantly impact how we handle output size limitations:
                            
 
                              
 
                            1. Contextual Awareness: Future LLMs will likely have improved contextual awareness - or as Mustafa Suleyman would call “infinite memory”, enabling them to better understand and manage the context of a conversation or document over long interactions. This will reduce the need for repetitive context setting and improve the overall user experience.
                              2. 
@@ -642,14 +642,18 @@ 
                              
-
                              2.6. Conclusion
                              
+
                              2.6. Conclusion
                              
 
                              In conclusion, while managing output size limitations in LLMs presents significant challenges, it also drives innovation in application design and optimization strategies. By implementing techniques such as context chunking, efficient prompt templates, and graceful fallbacks, developers can mitigate these limitations and enhance the performance and cost-effectiveness of their applications. As the technology evolves, advancements in contextual awareness, token efficiency, and memory management will further empower developers to build more robust and scalable LLM-powered systems. It is crucial to stay informed about these developments and continuously adapt to leverage the full potential of LLMs while addressing their inherent constraints.

                    -

                    2.7. References

                    - +

                    2.7. References

                    +
                    +
                    +[LangChain24] +

                    LangChain. Text splitters - langchain documentation. https://python.langchain.com/docs/how_to/#text-splitters, 2024. Accessed: 12/07/2024.

                    +
                    +
                    +

                  diff --git a/tamingllms/_build/html/notebooks/structured_output.html b/tamingllms/_build/html/notebooks/structured_output.html index dea2533..a432d0f 100644 --- a/tamingllms/_build/html/notebooks/structured_output.html +++ b/tamingllms/_build/html/notebooks/structured_output.html @@ -196,7 +196,7 @@
                  -

                  3. Wrestling with Structured Output

                  +

                  3. Wrestling with Structured Output

                  In limits, there is freedom. Creativity thrives within structure.

                  —Julia B. Cameron

                  @@ -204,42 +204,42 @@

                  Contents
                  -

                  3.1. Introduction

                  +

                  3.1. Introduction

                  Large language models (LLMs) excel at generating human-like text, but they often struggle to produce output in a structured format consistently. This poses a significant challenge when we need LLMs to generate data that can be easily processed by other systems, such as databases, APIs, or other software applications. Sometimes, even with a well-crafted prompt, an LLM might produce an unstructured response when a structured one is expected. This can be particularly challenging when integrating LLMs into systems that require specific data formats.

                  As a motivating example, consider the following simple task: Given a segment of a SEC financial filing, generate a two-person discussion about the key financial data from the text in JSON format, simulating what would be a real-world discussion about the underlying companies’ disclosed financial information. We would like to generate a structured output that can be easily parsed and integrated with other systems.

                  Throughout this notebook, we will consider as input a segment of a sample SEC filing of Apple Inc.

                  @@ -345,7 +345,7 @@

                  -

                  3.2. Problem Statement

                  +

                  3.2. Problem Statement

                  Obtaining structured output from LLMs presents several significant challenges:

                  • Inconsistency: LLMs often produce unpredictable results, sometimes generating well-structured output and other times deviating from the expected format.

                    • @@ -354,7 +354,7 @@

                    -

                    3.3. User Needs

                    +

                    3.3. User Needs

                    What user needs drive the demand for LLM output constraints when building LLM-based applications? In a recent work by Google Research [Liu et al., 2024], the authors explore the user need for constraints on the output of large language models, drawing on a survey of 51 industry professionals who use LLMs in their work. These needs can be broadly categorized as follows:

                    1. Improving Developer Efficiency and Workflow

                      @@ -377,10 +377,10 @@

                      -

                      3.4. Solutions

                      +

                      3.4. Solutions

                      Several strategies and tools can be employed to address the challenges of structured output from LLMs.

                      -

                      3.4.1. Strategies

                      +

                      3.4.1. Strategies

                      • Schema Guidance: Providing the LLM with a clear schema or blueprint of the desired output structure helps to constrain its generation and improve consistency. This can be achieved by using tools like Pydantic to define the expected data structure and then using that definition to guide the LLM’s output.

                      • Output Parsing: When LLMs don’t natively support structured output, parsing their text output using techniques like regular expressions or dedicated parsing libraries can extract the desired information. For example, you can use regular expressions to extract specific patterns from the LLM’s output, or you can use libraries like Pydantic to parse the output into structured data objects.

                        • @@ -388,9 +388,9 @@

                        -

                        3.4.2. Techniques and Tools

                        +

                        3.4.2. Techniques and Tools

                        -

                        3.4.2.1. One-Shot Prompts

                        +

                        3.4.2.1. One-Shot Prompts

                        In one-shot prompting, you provide a single example of the desired output format within the prompt.

                        @@ -457,7 +457,7 @@

                        -

                        3.4.2.2. Structured Output with Provider-Specific APIs

                        +

                        3.4.2.2. Structured Output with Provider-Specific APIs

                        One-shot prompting is a simple technique that can lead to material improvements in structured output, though may not be sufficient for complex (e.g. nested) structures and / or when the model’s output needs to be restricted to a specific set of options or types.

                        Provider-specific APIs can offer ways to handle those challenges. We will explore two approaches here using OpenAI’s API:

                          @@ -466,7 +466,7 @@

                          -

                          3.4.2.3. JSON Mode

                          +

                          3.4.2.3. JSON Mode

                          JSON mode is a feature provided by most LLM API providers, such as OpenAI, that allows the model to generate output in JSON format. This is particularly useful when you need structured data as a result, such as when parsing the output programmatically or integrating it with other systems that require JSON input. As depicted in Fig. 3.1, JSON mode is implemented by instructing theLLM model to use JSON as response format and optionally defining a target schema.

                          JSON Mode @@ -604,7 +604,7 @@

                          -

                          3.4.3. LangChain

                          +

                          3.4.3. LangChain

                          LangChain is a framework designed to simplify the development of LLM applications. It provider an abstraction layer over many LLM providers, including OpenAI, that offers several tools for parsing structured output.

                          In particular, LangChain offers the with_structured_output method, which can be used with LLMs that support structured output APIs, allowing you to enforce a schema directly within the prompt.

                          @@ -664,7 +664,7 @@

                          .with_structured_output() can be found here.

                        -

                        3.4.4. Outlines

                        +

                        3.4.4. Outlines

                        Outlines [Outlines, 2024] is a library specifically focused on structured text generation from LLMs. Under the hood, Outlines works by adjusting the probability distribution of the model’s output logits - the raw scores from the final layer of the neural network that are normally converted into text tokens. By introducing carefully crafted logit biases, Outlines can guide the model to prefer certain tokens over others, effectively constraining its outputs to a predefined set of valid options. This provides fine-grained control over the model’s generation process. In that way, Outlines provides several powerful features:

                        • Multiple Choice Generation: Restrict the LLM output to a predefined set of options.

                          • @@ -743,7 +743,7 @@

                          -

                          3.4.5. Ollama

                          +

                          3.4.5. Ollama

                          Ollama is a popular tool that allows you to run large language models (LLMs) locally. It has recently added support for structured output generation. The current ollama implementation leverages llama.cpp GBNF (GGML BNF) grammars [Ggerganov, 2024] to enable structured output generation. llama.cpp GBNF forces language models to generate output in specific, predefined formats by constraining their outputs to follow precise rules and patterns. The system accomplishes this through a formal grammar specification that defines exactly how valid outputs can be constructed. It’s essentially an extension of BNF (Backus-Naur Form) [Wikipedia contributors, 2024] with some modern regex-like features added. These rules carefully define what elements are allowed, how they can be combined, and what patterns of repetition and sequencing are valid. By enforcing these constraints during generation, GBNF ensures the model’s output strictly adheres to the desired format.

                          Ollama first introduced structured output generation in version 0.5.1 providing support for JSON output but highlighting additional formats are coming soon.

                          Let’s replicate our previous structured output generation example with Ollama. First, make sure you have Ollama installed. You can find installation instructions here.

                          @@ -840,9 +840,9 @@

                          -

                          3.5. Discussion

                          +

                          3.5. Discussion

                          -

                          3.5.1. Comparing Solutions

                          +

                          3.5.1. Comparing Solutions

                          The choice of framework for structured LLM output depends heavily on specific constraints, requirements and use cases. LangChain is the most used LLM framework today with a large developer community base however its structured output support depends on the underlying LLM provider support. Ollama enables straightforward local deployment and experimentation democratizing access to LLMs while fostering privacy and control, however today it only offers JSON format with further formats to come. Outlines emerges as a solution with great flexibility and control over output structure while providing support for a wide range of LLMs. Table 3.1 provides a summary comparison of the different frameworks.

              @@ -888,7 +888,7 @@

              -

              3.5.2. Best Practices

              +

              3.5.2. Best Practices

              • Clear Schema Definition: Define the desired output structure clearly. This can be done in several ways including schemas, types, or Pydantic models as appropriate. This ensures the LLM knows exactly what format is expected.

              • Descriptive Naming: Use meaningful names for fields and elements in your schema. This makes the output more understandable and easier to work with.

                • @@ -897,7 +897,7 @@

                -

                3.5.3. Research and Ongoing Debate

                +

                3.5.3. Research and Ongoing Debate

                The use of structured output for Large Language Models (LLMs) is a developing area. While the ability to constrain LLM outputs offer clear benefits in parsing, robustness, and integration, there is growing debate on whether it also potentially comes at the cost of performance as well as reasoning abilities. Research in this area should be taken with a grain of salt since findings are mixed and often depend on the specific task and model family at hand furthermore model families are not always comparable and are getting updated by the day! Nonetheless, early findings provide some interesting insights as to why there is no one-size-fits-all solution when it comes to LLMs structured output.

                There is some evidence indicating that LLMs may have bias in their handling of different output formats [Long et al., 2024]. The study examined common output structures like multiple-choice answers, wrapped text, lists, and key-value mappings. The authors analyzed key LLM model families, namely Gemma, Mistral, and ChatGPT, uncovering bias across multiple tasks and formats. The researchers attributed these biases to the models’ underlying token distributions for different formats. An example of this format bias emerged in the comparison between JSON and YAML outputs. While models like Mistral and Gemma excelled at generating JSON structures, they performed notably worse with YAML. Their YAML outputs often contained extraneous information that degrades output quality. This disparity likely stems from JSON’s prevalence in training data, highlighting how a format’s popularity directly influences model performance. While the studied models can be probably considered outdated by now since models are getting updated on a rapidly fashion, it is important to remark that addressing format bias is critical for advancing LLMs and ensuring their reliable application in real-world scenarios.

                Recent research “Let Me Speak Freely? A Study on the Impact of Format Restrictions on Performance of Large Language Models” [Tam et al., 2024] suggests that imposing format restrictions on LLMs might impact their performance, particularly in reasoning-intensive tasks. Further evidence [Aider, 2024] suggests LLMs may produce lower quality code if they’re asked to return it as part of a structured JSON response, in particular:

                @@ -927,15 +927,15 @@

                -

                3.6. Conclusion

                +

                3.6. Conclusion

                Extracting structured output from LLMs is crucial for integrating them into real-world applications. By understanding the challenges and employing appropriate strategies and tools, developers can improve the reliability and usability of LLM-powered systems, unlocking their potential to automate complex tasks and generate valuable insights.

                -

                3.7. Acknowledgements

                +

                3.7. Acknowledgements

                We would like to thank Cameron Pfiffer from the .txt team for his insightful review and feedback.

                -

                3.8. References

                +

                3.8. References

                [Aid24] diff --git a/tamingllms/_build/html/searchindex.js b/tamingllms/_build/html/searchindex.js index 11d1f52..d6a8f82 100644 --- a/tamingllms/_build/html/searchindex.js +++ b/tamingllms/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/evals", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/evals.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["1. Introduction", "Taming LLMs", "4. The Evals Gap", "2. Output Size Limitations", "3. Wrestling with Structured Output"], "terms": {"am": 0, "alwai": [0, 2, 4], "do": [0, 2, 3, 4], "which": [0, 2, 3, 4], "cannot": [0, 2], "order": [0, 2, 4], "mai": [0, 2, 3, 4], "learn": [0, 2], "how": [0, 2, 3, 4], "pablo": 0, "picasso": 0, "In": [0, 2, 3, 4], "recent": [0, 4], "year": [0, 2, 3, 4], "larg": [0, 1, 2, 3, 4], "languag": [0, 1, 2, 3, 4], "model": [0, 1, 4], "llm": [0, 3, 4], "have": [0, 2, 3, 4], "emerg": [0, 1, 4], "transform": [0, 2, 4], "forc": [0, 2, 4], "technologi": [0, 2, 3, 4], "promis": 0, "revolution": 0, "build": [0, 1, 2, 3, 4], "product": [0, 1, 2, 4], "interact": [0, 2, 3, 4], "comput": [0, 2, 3, 4], "from": [0, 2, 3, 4], "chatgpt": [0, 2, 4], "github": [0, 2, 4], "copilot": 0, "claud": [0, 2, 3], "artifact": 0, "system": [0, 2, 3, 4], "captur": [0, 2], "public": [0, 2], "imagin": 0, "spark": 0, "gold": [0, 2], "rush": 0, "ai": [0, 2, 4], "power": [0, 1, 2, 3, 4], "applic": [0, 1, 3, 4], "howev": [0, 2, 3, 4], "beneath": 0, "surfac": [0, 2], "technolog": 0, "revolut": 0, "li": [0, 2], "complex": [0, 2, 3, 4], "landscap": [0, 2], "practition": [0, 2], "must": [0, 2, 3], "navig": [0, 1], "focus": [0, 2, 3, 4], "bring": 0, "awar": [0, 2, 3], "limit": [0, 2, 4], "har": [0, 1, 3], "open": [0, 2, 3, 4], "sourc": [0, 2, 4], "solut": [0, 1, 2, 3], "overcom": [0, 3], "them": [0, 2, 3, 4], "robust": [0, 2, 3, 4], "It": [0, 2, 3, 4], "offer": [0, 2, 4], "critic": [0, 1, 2, 3, 4], "implement": [0, 1, 2, 3, 4], "back": [0, 4], "reproduc": [0, 1, 2], "exampl": [0, 1, 2, 4], "while": [0, 1, 2, 3, 4], "mani": [0, 3, 4], "resourc": [0, 2, 3], "cover": [0, 2, 3], "capabl": [0, 1, 2, 3, 4], "specif": [0, 1, 2, 3], "hidden": 0, "pitfal": 0, "engin": [0, 1, 2, 4], "technic": [0, 1, 2, 3, 4], "manag": [0, 1, 2, 3], "face": [0, 2, 4], "when": [0, 1, 2, 3, 4], "comprehens": [0, 1, 2, 3, 4], "guid": [0, 2, 4], "leverag": [0, 2, 3, 4], "battl": [0, 1], "test": [0, 1, 4], "tool": [0, 3], "throughout": [0, 3, 4], "tackl": [0, 2], "follow": [0, 2, 3, 4], "non": [0, 1, 4], "exhaust": 0, "list": [0, 2, 3, 4], "structur": [0, 2, 3], "un": 0, "reliabl": [0, 2, 4], "struggl": [0, 2, 4], "maintain": [0, 2, 3, 4], "consist": [0, 2, 3, 4], "output": [0, 2], "format": [0, 2, 3, 4], "complic": 0, "integr": [0, 2, 4], "larger": [0, 2, 3, 4], "make": [0, 2, 3, 4], "error": [0, 2, 4], "handl": [0, 1, 2, 3, 4], "more": [0, 2, 3, 4], "size": [0, 2, 4], "length": [0, 2, 4], "constraint": [0, 1, 3, 4], "strict": [0, 4], "token": [0, 1, 2, 4], "both": [0, 2], "input": [0, 2, 3, 4], "requir": [0, 3, 4], "care": [0, 2, 4], "chunk": [0, 1], "strategi": [0, 1, 2, 3], "long": [0, 1, 2, 4], "form": [0, 1, 2, 4], "effect": [0, 2, 3, 4], "tradit": 0, "softwar": [0, 4], "methodologi": [0, 2, 4], "break": [0, 2, 3], "down": [0, 2, 3], "deal": 0, "determinist": [0, 1, 4], "gener": [0, 1, 4], "new": [0, 2, 3, 4], "hallucin": [0, 2, 4], "These": [0, 2, 3, 4], "can": [0, 2, 3, 4], "plausibl": 0, "sound": 0, "entir": [0, 2, 3], "fabric": [0, 2], "inform": [0, 2, 3, 4], "creat": [0, 2, 3, 4], "signific": [0, 2, 3, 4], "risk": [0, 2, 3], "safeti": [0, 2, 4], "secur": [0, 2, 3, 4], "harm": [0, 2], "bias": [0, 2, 4], "inappropri": 0, "safeguard": [0, 2], "monitor": [0, 1], "ensur": [0, 2, 3, 4], "safe": [0, 4], "deploy": [0, 1, 2, 4], "cost": [0, 2, 4], "optim": [0, 1, 2, 3], "The": [0, 3, 4], "financi": [0, 2, 3, 4], "oper": [0, 2, 3], "base": [0, 1, 4], "quickli": [0, 3], "becom": [0, 2], "prohibit": 0, "without": [0, 2, 3, 4], "observ": [0, 2, 4], "vendor": [0, 1, 2], "lock": [0, 1], "cloud": [0, 2, 4], "provid": [0, 2, 3], "depend": [0, 2, 4], "through": [0, 1, 2, 3, 4], "proprietari": [0, 4], "infrastructur": 0, "difficult": [0, 2], "switch": 0, "self": [0, 1, 2], "host": [0, 1, 2], "take": [0, 1, 2, 3, 4], "hand": [0, 3, 4], "concret": [0, 1], "you": [0, 2, 3, 4], "run": [0, 2, 4], "modifi": 0, "real": [0, 2, 3, 4], "world": [0, 2, 4], "scenario": [0, 2, 4], "best": [0, 1, 2], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 2, 4], "anti": 0, "look": [0, 1, 2], "our": [0, 2, 3, 4], "goal": [0, 2, 3], "discourag": 0, "us": [0, 3, 4], "enabl": [0, 2, 3, 4], "By": [0, 1, 2, 3, 4], "understand": [0, 1, 2, 3, 4], "upfront": [0, 1], "better": [0, 1, 2, 3], "equip": [0, 1], "avoid": [0, 2, 4], "current": [0, 1, 3, 4], "discours": [0, 1], "around": [0, 1, 2, 3, 4], "tend": [0, 1, 2], "toward": [0, 2, 4], "extrem": 0, "either": [0, 2, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": 0, "dismiss": 0, "differ": [0, 2, 3, 4], "focu": [0, 1, 2, 3, 4], "rather": [0, 2], "than": [0, 2], "theoret": 0, "examin": [0, 3, 4], "first": [0, 2, 3, 4], "everi": 0, "concept": [0, 2], "illustr": [0, 2, 3], "execut": [0, 2], "immedi": 0, "analysi": [0, 1, 2, 3], "balanc": [0, 2, 3, 4], "help": [0, 2, 3, 4], "reader": [0, 1], "decis": [0, 2, 4], "intend": 0, "develop": [0, 2, 3, 4], "step": [0, 1, 2, 4], "insight": [0, 2, 3, 4], "along": [0, 2], "guidanc": [0, 4], "framework": [0, 2], "could": [0, 2, 3, 4], "derail": 0, "project": 0, "earli": [0, 2, 4], "befor": [0, 2, 4], "thei": [0, 2, 3, 4], "costli": 0, "problem": [0, 1], "too": [0, 2, 3], "late": 0, "lifecycl": 0, "design": [0, 1, 3, 4], "lead": [0, 2, 3, 4], "genai": 0, "initi": [0, 2, 3], "leader": 0, "architectur": [0, 2, 3], "advoc": 0, "anyon": 0, "seek": [0, 2], "work": [0, 1, 2, 3, 4], "typic": [0, 2, 3], "job": 0, "role": [0, 2, 3, 4], "platform": [0, 2, 3, 4], "backend": [0, 2], "exist": [0, 2], "ml": 0, "transit": [0, 2, 3], "overse": 0, "motiv": [0, 2, 4], "need": [0, 2, 3], "readi": [0, 2], "desir": [0, 2, 4], "perform": [0, 1, 2, 3, 4], "after": [0, 2, 3], "read": [0, 2, 3, 4], "implic": [0, 1, 2], "experi": [0, 2, 3, 4], "recommend": [0, 2, 3, 4], "abl": [0, 3, 4], "deploi": [0, 3], "proper": [0, 4], "realist": 0, "effort": [0, 2, 4], "estim": [0, 2], "impact": [0, 2, 3, 4], "timelin": 0, "To": [0, 2, 3, 4], "most": [0, 2, 3, 4], "should": [0, 2, 3, 4], "basic": [0, 2, 3], "program": [0, 2], "knowledg": [0, 2], "introductori": [0, 1], "langchain": [0, 1, 3], "e": [0, 2, 3, 4], "g": [0, 2, 3, 4], "chat": [0, 2, 3, 4], "prompt": [0, 1, 2], "templat": [0, 1, 2], "access": [0, 2, 4], "openai": [0, 2, 4], "anthrop": [0, 4], "similar": [0, 2, 4], "grade": 0, "dive": 0, "here": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": [0, 2, 4], "activ": [0, 2], "virtual": [0, 2], "m": 0, "venv": 0, "env": [0, 2, 3, 4], "bin": 0, "On": [0, 4], "window": [0, 1], "script": 0, "instal": [0, 2, 4], "packag": 0, "pip": [0, 2, 4], "r": [0, 2, 3, 4], "txt": [0, 2, 3, 4], "file": [0, 2, 3, 4], "root": 0, "directori": [0, 2], "add": [0, 3], "other": [0, 2, 3, 4], "sensit": [0, 2], "openai_api_kei": 0, "your_openai_api_key_her": 0, "never": 0, "share": [0, 2, 4], "commit": [0, 2], "version": [0, 2, 4], "control": [0, 2, 4], "contain": [0, 2, 3, 4], "kept": [0, 2], "privat": 0, "clone": 0, "companion": 0, "git": 0, "http": [0, 2, 4], "com": [0, 2, 4], "souzatharsi": 0, "tamingllm": 0, "cd": 0, "If": [0, 2, 4], "encount": [0, 1, 2], "rate": [0, 2], "consid": [0, 2, 3, 4], "smaller": [0, 2, 3, 4], "retri": [0, 4], "logic": [0, 2, 3], "conflict": 0, "try": [0, 2, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": [0, 2], "page": 0, "known": [0, 2, 4], "now": [0, 2, 3, 4], "let": [0, 2, 3, 4], "begin": [0, 2], "explor": [0, 2, 4], "dr": 0, "tharsi": 0, "souza": 0, "scientist": 0, "special": [0, 2, 4], "he": [0, 2], "lectur": 0, "columbia": 0, "univers": 0, "master": [0, 4], "scienc": [0, 2], "appli": [0, 2, 3], "analyt": 0, "head": [0, 3], "equiti": 0, "citadel": 0, "former": [0, 2], "senior": 0, "vp": 0, "two": [0, 2, 3, 4], "sigma": 0, "invest": [0, 2, 4], "With": 0, "over": [0, 1, 2, 3, 4], "15": [0, 2, 4], "deliv": 0, "across": [0, 2, 4], "startup": 0, "fortun": 0, "500": [0, 2], "compani": [0, 2, 3, 4], "global": 0, "also": [0, 2, 3, 4], "an": [0, 1, 2, 3, 4], "numer": [0, 2], "scholarli": 0, "frequent": [0, 4], "speaker": 0, "academ": [0, 2], "busi": [0, 2], "confer": [0, 4], "ground": [0, 1, 2], "background": [0, 3], "draw": [0, 4], "scale": [0, 2, 4], "stage": 0, "major": [0, 2, 4], "institut": 0, "well": [0, 2, 4], "advis": 0, "profit": [0, 2, 3, 4], "organ": [0, 2, 3], "contribut": [0, 2, 3], "uniqu": [0, 2], "bridg": 0, "gap": 0, "between": [0, 2, 3, 4], "potenti": [0, 2, 3, 4], "next": [0, 2, 4], "hold": 0, "ph": 0, "d": [0, 4], "ucl": 0, "london": 0, "phil": 0, "sc": 0, "b": [0, 2, 4], "abstract": [1, 2, 4], "heavili": [1, 2, 4], "gloss": 1, "fundament": [1, 2, 4], "challeng": [1, 2, 3, 4], "convers": [1, 2, 3, 4], "thi": [1, 2, 3, 4], "book": 1, "kei": [1, 4], "python": [1, 2, 4], "proven": 1, "yet": [1, 2, 3], "i": [1, 2, 3, 4], "unstructur": [1, 4], "context": [1, 2, 3, 4], "code": [1, 2, 4], "sidestep": 1, "inher": [1, 2, 3, 4], "core": [1, 2], "we": [1, 2, 3, 4], "ll": [1, 2], "address": [1, 2, 3, 4], "approach": [1, 2, 3, 4], "note": [1, 2, 3, 4], "perspect": 1, "who": [1, 2, 3, 4], "For": [1, 2, 3, 4], "outcom": [1, 2, 4], "prerequisit": 1, "set": [1, 2, 3, 4], "up": [1, 2, 3, 4], "your": [1, 2, 3, 4], "environ": [1, 2, 3, 4], "setup": [1, 2, 4], "api": [1, 2], "configur": [1, 2], "repositori": [1, 2], "troubleshoot": 1, "common": [1, 2, 3, 4], "issu": [1, 2, 3, 4], "about": [1, 2, 3, 4], "author": [1, 4], "": [1, 2, 3, 4], "statement": 1, "One": [1, 2], "shot": [1, 2], "json": [1, 2], "mode": 1, "outlin": [1, 2], "multipl": [1, 2, 3, 4], "choic": [1, 2, 4], "pydant": [1, 2, 4], "discuss": [1, 2], "compar": [1, 2, 3], "research": [1, 2, 3], "ongo": [1, 2], "debat": 1, "conclus": [1, 2], "acknowledg": [1, 2], "refer": 1, "content": 1, "what": [1, 2, 4], "ar": [1, 2, 4], "contextu": [1, 2], "link": 1, "write": [1, 2, 4], "construct": [1, 2, 4], "dynam": [1, 2], "paramet": [1, 2, 4], "report": [1, 2, 4], "usag": [1, 2, 4], "futur": [1, 2], "consider": [1, 4], "machin": 1, "temperatur": [1, 3, 4], "sampl": [1, 3, 4], "spectrum": 1, "properti": 1, "conceptu": [1, 4], "overview": [1, 4], "compon": [1, 2], "metric": 1, "evalu": [1, 3, 4], "human": [1, 3, 4], "benchmark": 1, "leaderboard": 1, "type": [1, 2, 3, 4], "detect": [1, 2, 4], "retriev": [1, 2], "augment": [1, 2], "rag": 1, "select": [1, 2], "index": [1, 2, 3], "vector": 1, "store": [1, 2, 3], "method": [1, 2, 3, 4], "pipelin": [1, 2, 4], "valid": [1, 2, 4], "guard": 1, "filter": [1, 2], "sanit": 1, "alert": 1, "cach": [1, 2], "invalid": [1, 4], "predict": [1, 2, 4], "llama": [1, 2, 4], "llamafil": 1, "ollama": 1, "migrat": 1, "commun": [1, 2, 4], "surprisingli": 2, "often": [2, 3, 4], "all": [2, 3, 4], "greg": 2, "brockman": 2, "presid": 2, "natur": [2, 3, 4], "unlik": 2, "where": [2, 3, 4], "same": [2, 3, 4], "produc": [2, 4], "respons": [2, 3, 4], "each": [2, 3], "time": [2, 3, 4], "re": [2, 3, 4], "queri": 2, "even": [2, 3, 4], "ident": 2, "data": [2, 3, 4], "characterist": 2, "strength": 2, "ask": [2, 4], "ani": [2, 3, 4], "question": [2, 4], "isn": 2, "t": [2, 3, 4], "bug": 2, "featur": [2, 4], "random": 2, "allow": [2, 3, 4], "creativ": [2, 4], "divers": [2, 3, 4], "incredibli": 2, "testabl": 2, "servic": [2, 3, 4], "advic": 2, "mean": [2, 3, 4], "market": [2, 3, 4], "yield": 2, "exceedingli": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3], "trust": [2, 4], "affect": 2, "inconsist": [2, 4], "primari": 2, "determin": [2, 3, 4], "come": [2, 3, 4], "dure": [2, 4], "text": [2, 3, 4], "calcul": 2, "probabl": [2, 4], "distribut": [2, 4], "nucleu": 2, "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3, 4], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "dotenv": [2, 3, 4], "import": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "variabl": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 4], "datafram": 2, "demonstr": [2, 3, 4], "behavior": 2, "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3, 4], "complet": [2, 3, 4], "messag": [2, 4], "max_token": 2, "50": 2, "append": [2, 3, 4], "displai": [2, 4], "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": 2, "row": 2, "iterrow": 2, "return": [2, 3, 4], "max_length": [2, 4], "10000": [2, 3, 4], "appl": [2, 3, 4], "sec_fil": [2, 4], "gpt": [2, 3, 4], "5": [2, 3, 4], "turbo": [2, 3, 4], "singl": [2, 3, 4], "summari": [2, 4], "2": [2, 4], "inc": [2, 3, 4], "its": [2, 3, 4], "10": [2, 3, 4], "k": [2, 3, 4], "fiscal": [2, 3], "end": [2, 3], "septemb": [2, 3], "28": [2, 3], "2024": [2, 3, 4], "sec": [2, 3, 4], "detail": [2, 3, 4], "season": 2, "issuer": 2, "california": [2, 4], "manufactur": 2, "smartphon": 2, "person": [2, 4], "tablet": 2, "wearabl": [2, 4], "accessori": 2, "innov": [2, 3], "condit": 2, "exchang": [2, 3, 4], "commiss": [2, 3, 4], "factor": [2, 3, 4], "invdestacksmeticsisdict": 2, "setispect": 2, "20cyan": 2, "evaluationseld": 2, "anvis": 2, "droitent": 2, "discernminerv": 2, "versbobprefvers": 2, "vo\u8be5": 2, "option\u548c": 2, "meio": 2, "forecast": 2, "\u0432\u0440\u0435\u043ccisco": 2, "dellaischenpoihscap": 2, "geme": 2, "gettim": 2, "simpl": [2, 3, 4], "reveal": 2, "dramat": [2, 4], "alter": 2, "wai": [2, 3, 4], "systemat": [2, 4], "assess": [2, 3], "At": 2, "rigid": 2, "vari": 2, "less": 2, "wildli": 2, "inadequ": 2, "profound": 2, "one": [2, 3, 4], "radic": 2, "reli": 2, "u": [2, 4], "grappl": 2, "probabilist": 2, "lower": [2, 4], "seem": [2, 4], "safer": 2, "don": [2, 3, 4], "elimin": 2, "underli": [2, 4], "uncertainti": 2, "mere": 2, "mask": 2, "highlight": [2, 3, 4], "paradigm": 2, "aspect": [2, 3, 4], "beyond": 2, "present": [2, 3, 4], "anoth": 2, "fascin": 2, "abil": [2, 4], "spontan": 2, "aris": 2, "answer": [2, 3, 4], "reason": [2, 3, 4], "aren": 2, "explicitli": 2, "grow": [2, 4], "train": [2, 4], "against": 2, "clear": [2, 4], "wei": 2, "et": [2, 4], "al": [2, 4], "2022": 2, "fig": [2, 3, 4], "4": 2, "relationship": 2, "linear": 2, "below": [2, 3, 4], "certain": [2, 3, 4], "threshold": 2, "absent": 2, "simpli": [2, 3, 4], "matter": 2, "much": 2, "coax": 2, "out": [2, 3], "onc": [2, 3], "reach": [2, 3, 4], "point": [2, 3], "journei": 2, "suddenli": 2, "manifest": 2, "call": [2, 3, 4], "phase": 2, "shift": 2, "inabl": 2, "unpredict": [2, 4], "stand": 2, "stark": 2, "contrast": 2, "deliber": 2, "convent": 2, "stabl": 2, "suit": 2, "defin": [2, 3, 4], "accept": 2, "criteria": 2, "contend": 2, "constantli": 2, "7b": 2, "70b": 2, "ha": [2, 4], "rethink": 2, "practic": [2, 3], "math": 2, "tutor": 2, "children": 2, "would": [2, 3, 4], "verifi": [2, 4], "function": [2, 3, 4], "But": [2, 4], "just": [2, 3, 4], "predefin": [2, 4], "adapt": [2, 3], "explan": [2, 4], "child": 2, "level": [2, 3, 4], "engag": [2, 4], "ag": 2, "appropri": [2, 3, 4], "encourag": [2, 4], "emot": 2, "support": [2, 4], "rais": [2, 3], "measur": 2, "weren": 2, "evolv": [2, 3], "accuraci": [2, 4], "subject": 2, "qualiti": [2, 3, 4], "kind": 2, "There": [2, 3, 4], "account": 2, "tabl": [2, 3, 4], "sever": [2, 3, 4], "dimens": 2, "necessarili": 2, "pre": 2, "extend": [2, 4], "explicit": [2, 4], "usual": 2, "precis": [2, 4], "involv": [2, 4], "resist": 2, "straightforward": [2, 3, 4], "quantif": 2, "score": [2, 4], "judgment": 2, "remain": [2, 3], "contamin": 2, "carefulli": [2, 4], "craft": [2, 4], "case": [2, 3, 4], "expect": [2, 3, 4], "unit": [2, 3, 4], "massiv": 2, "internet": 2, "alreadi": 2, "seen": 2, "memor": 2, "artifici": 2, "inflat": 2, "curat": 2, "truli": 2, "unseen": 2, "rigor": 2, "cross": 2, "evolut": 2, "continu": [2, 3, 4], "advanc": [2, 3, 4], "longitudin": 2, "comparison": 2, "obsolet": 2, "older": 2, "autom": [2, 4], "demand": [2, 4], "oversight": 2, "annot": 2, "review": [2, 4], "process": [2, 3, 4], "mostli": [2, 4], "distinct": 2, "versu": 2, "latter": 2, "foundat": [2, 3], "purpos": [2, 4], "tailor": 2, "particular": [2, 4], "combin": [2, 3, 4], "associ": [2, 3, 4], "solv": [2, 4], "That": [2, 4], "differenti": 2, "becaus": 2, "chang": 2, "scope": [2, 3], "includ": [2, 3, 4], "thing": [2, 4], "instead": [2, 3, 4], "meet": [2, 4], "align": [2, 3, 4], "object": [2, 4], "A": [2, 3, 4], "great": [2, 4], "doesn": [2, 3, 4], "categori": 2, "why": [2, 4], "misinform": 2, "prevent": [2, 4], "factual": 2, "databas": [2, 4], "citat": 2, "tempor": 2, "scientif": 2, "fals": [2, 4], "reduc": [2, 3, 4], "legal": 2, "reput": 2, "protect": 2, "manipul": 2, "unqualifi": 2, "recognit": 2, "medic": 2, "disclaim": 2, "profession": [2, 4], "referr": 2, "mechan": 2, "boundari": 2, "situat": [2, 3], "incorrect": 2, "liabil": 2, "vulner": 2, "standard": 2, "expertis": 2, "util": [2, 3], "bia": [2, 4], "gender": 2, "racial": 2, "cultur": 2, "demograph": 2, "represent": [2, 3], "inclus": [2, 3, 4], "stereotyp": 2, "fair": 2, "reinforc": 2, "societ": 2, "equal": 2, "social": 2, "brand": 2, "privaci": [2, 4], "pii": 2, "anonym": 2, "leakag": 2, "carryov": 2, "regul": [2, 4], "protocol": 2, "confidenti": 2, "breach": 2, "cognit": 2, "multi": [2, 4], "mathemat": 2, "fallaci": 2, "causal": 2, "edg": 2, "think": 2, "mainten": 2, "idiom": 2, "sarcasm": 2, "terminologi": 2, "lingual": 2, "misunderstand": 2, "sophist": [2, 3], "syntax": 2, "scan": 2, "document": [2, 3, 4], "compat": [2, 4], "stabil": 2, "effici": [2, 3, 4], "debt": 2, "scalabl": [2, 3], "failur": 2, "meta": [2, 3], "correct": [2, 4], "feedback": [2, 4], "overconfid": 2, "improv": [2, 3, 4], "clariti": [2, 3, 4], "audienc": 2, "densiti": 2, "transfer": 2, "satisfact": [2, 4], "ethic": 2, "request": [2, 3, 4], "incid": 2, "misus": 2, "moral": 2, "valu": [2, 3, 4], "transpar": [2, 4], "stakehold": 2, "environment": 2, "co2": 2, "emiss": 2, "energi": 2, "consumpt": 2, "per": [2, 3], "server": [2, 4], "locat": 2, "batch": 2, "hardwar": 2, "infer": 2, "sustain": 2, "corpor": 2, "three": 2, "app": 2, "imag": 2, "audio": 2, "etc": [2, 4], "truth": [2, 4], "option": [2, 3, 4], "layer": [2, 3, 4], "repres": [2, 4], "palm": 2, "individu": [2, 3], "target": [2, 4], "further": [2, 3, 4], "see": [2, 4], "avail": [2, 3, 4], "addition": 2, "shown": 2, "fix": [2, 3], "default": [2, 4], "quantifi": 2, "rank": 2, "easi": [2, 3], "addit": [2, 3, 4], "quantit": 2, "among": 2, "aggreg": 2, "plan": [2, 4], "pertain": 2, "previous": [2, 3, 4], "doe": [2, 3, 4], "good": [2, 4], "separ": [2, 3], "synthet": [2, 4], "updat": [2, 3, 4], "reflect": 2, "post": [2, 4], "launch": 2, "timeout": 2, "variat": 2, "maxim": 2, "success": [2, 4], "inter": 2, "rater": 2, "weight": 2, "rel": 2, "priorit": 2, "normal": [2, 4], "absolut": [2, 4], "fail": 2, "confid": [2, 4], "interv": 2, "ti": 2, "veri": 2, "close": 2, "tier": 2, "holist": 2, "built": [2, 4], "mind": 2, "x": 2, "fast": 2, "particularli": [2, 3, 4], "promot": 2, "rapid": 2, "experiment": [2, 4], "iter": [2, 3, 4], "final": [2, 3, 4], "keep": [2, 3], "itself": 2, "confirm": 2, "vi": 2, "later": [2, 4], "chapter": 2, "categor": [2, 4], "broad": [2, 4], "intrins": 2, "extrins": 2, "sequenc": [2, 4], "perplex": 2, "given": [2, 3, 4], "variou": [2, 3, 4], "downstream": [2, 4], "directli": [2, 4], "valuabl": [2, 4], "interest": [2, 3, 4], "sinc": [2, 3, 4], "term": [2, 3], "discrimin": 2, "distinguish": 2, "classifi": 2, "sentiment": [2, 4], "classif": [2, 4], "identifi": [2, 3, 4], "whether": [2, 3, 4], "true": [2, 3, 4], "synthesi": 2, "summar": [2, 3], "log": 2, "discret": 2, "recal": 2, "f1": 2, "match": [2, 4], "exact": 2, "prefix": 2, "translat": 2, "roug": 2, "bleu": 2, "charact": [2, 3, 4], "n": [2, 3], "gram": 2, "being": [2, 4], "guidelin": 2, "subsequ": [2, 4], "section": [2, 3, 4], "short": [2, 3, 4], "wide": [2, 3, 4], "definit": [2, 4], "bilingu": 2, "understudi": 2, "overlap": [2, 3], "favor": [2, 4], "due": [2, 3], "breviti": 2, "penalti": 2, "insensit": 2, "semant": [2, 3], "high": [2, 3], "orient": 2, "gist": 2, "word": [2, 3, 4], "sentenc": [2, 3, 4], "ignor": 2, "equival": 2, "influenc": [2, 4], "meteor": 2, "synonym": 2, "stem": [2, 4], "paraphras": 2, "alongsid": 2, "computation": [2, 3], "expens": 2, "cider": 2, "consensu": 2, "descript": [2, 4], "tf": 2, "idf": 2, "caption": 2, "outsid": 2, "reliant": 2, "corpu": 2, "statist": 2, "ter": 2, "edit": 2, "number": [2, 3, 4], "convert": [2, 4], "hypothesi": 2, "penal": 2, "bertscor": 2, "embed": [2, 3], "bert": 2, "spice": 2, "proposit": 2, "scene": 2, "graph": 2, "emphasi": 2, "onli": [2, 3, 4], "pure": 2, "textual": 2, "As": [2, 3, 4], "analyst": [2, 3], "prepar": [2, 3], "dictionari": [2, 4], "rouge_1": 2, "rouge_2": 2, "ideal": [2, 4], "humana": 2, "expert": [2, 3, 4], "cheaper": 2, "4o": [2, 3, 4], "mini": [2, 3, 4], "evaluate_summari": 2, "unigram": 2, "bigram": 2, "huggingfac": 2, "librari": [2, 3, 4], "absl": 2, "py": 2, "rouge_scor": 2, "generated_summari": 2, "reference_summari": 2, "arg": [2, 3, 4], "dict": [2, 3, 4], "google_bleu": 2, "bleu_scor": 2, "rouge1": 2, "rouge2": 2, "instanc": [2, 3], "arbitrari": 2, "chosen": 2, "sentence1": 2, "cat": 2, "sat": 2, "mat": 2, "sentence2": 2, "ate": 2, "3333333333333333": 2, "7272727272727272": 2, "4444444444444445": 2, "generate_summari": 2, "summir": 2, "correspond": [2, 4], "liner": 2, "excerpt": 2, "evaluate_summary_model": 2, "model_benchmark": 2, "models_test": 2, "benchmark_summari": 2, "model_summari": 2, "evaluation_result": 2, "line": 2, "name": [2, 3, 4], "zip": 2, "annual": 2, "stock": [2, 4], "govern": 2, "forward": 2, "analyz": [2, 3, 4], "statu": 2, "concis": 2, "omit": [2, 4], "essenti": [2, 3, 4], "element": [2, 4], "Its": 2, "adequ": 2, "verbos": 2, "relev": 2, "introduc": [2, 3, 4], "peripher": 2, "quit": [2, 4], "overli": [2, 4], "simplifi": [2, 4], "miss": 2, "convei": [2, 3], "breadth": 2, "Of": 2, "cours": 2, "abov": 2, "vibe": 2, "visualize_prompt_comparison": 2, "visual": 2, "matplotlib": 2, "radar": 2, "plot": 2, "radar_plot": 2, "show": [2, 3, 4], "tmp": 2, "ipykernel_1652501": 2, "940173201": 2, "userwarn": 2, "figurecanvasagg": 2, "thu": 2, "put": 2, "closest": 2, "largest": 2, "deviat": [2, 4], "suggest": [2, 4], "least": 2, "establish": 2, "otherwis": 2, "qualit": 2, "driven": 2, "mention": [2, 4], "might": [2, 3, 4], "fulli": [2, 3], "nuanc": [2, 3, 4], "especi": [2, 3, 4], "those": [2, 3, 4], "primarili": 2, "granular": [2, 3], "altern": [2, 3], "fall": 2, "judg": 2, "themselv": 2, "act": 2, "paper": [2, 4], "main": [2, 3, 4], "instruct": [2, 3, 4], "fine": [2, 4], "tune": [2, 4], "assign": 2, "likert": 2, "style": 2, "pairwis": 2, "ensembl": 2, "repeatedli": 2, "domain": 2, "procedur": 2, "fluenci": 2, "interpret": 2, "refin": 2, "excel": [2, 4], "narr": 2, "flow": [2, 3], "mirror": 2, "express": [2, 4], "similarli": 2, "notabl": [2, 4], "properli": [2, 4], "henc": 2, "worth": 2, "integ": 2, "rubric": 2, "foster": [2, 4], "hollist": 2, "judgeevalu": 2, "enforc": [2, 4], "four": 2, "grammar": [2, 4], "evaluate_with_llm": 2, "candid": 2, "pars": [2, 4], "criterion": 2, "basemodel": [2, 4], "class": [2, 3, 4], "judge_model": 2, "candidate_summari": 2, "specifi": [2, 3, 4], "wa": [2, 4], "written": 2, "grammat": 2, "y": 2, "z": 2, "w": [2, 3], "beta": [2, 4], "response_format": [2, 4], "Then": 2, "benchmark_model": 2, "test_model": 2, "input_text": [2, 3], "tupl": 2, "iphon": [2, 4], "mac": [2, 4], "ipad": [2, 4], "incorpor": 2, "regard": 2, "obtain": [2, 4], "respect": 2, "regist": 2, "approxim": [2, 4], "6": [2, 3, 4], "trillion": [2, 4], "held": [2, 4], "affili": [2, 4], "billion": 2, "outstand": [2, 4], "octob": [2, 4], "18": [2, 4], "7": [2, 3], "8": [2, 3], "evals_list": 2, "1775618912": 2, "14": [2, 4], "some": [2, 3, 4], "achiev": [2, 4], "variant": 2, "slightli": 2, "indic": [2, 4], "drift": 2, "had": 2, "lowest": 2, "overal": [2, 3, 4], "drop": 2, "substanti": 2, "gradient": 2, "visibl": 2, "mark": 2, "degrad": [2, 4], "firstli": 2, "overhead": 2, "neglect": 2, "exhibit": 2, "prefer": [2, 4], "posit": [2, 3, 4], "egocentr": 2, "tight": 2, "small": [2, 4], "field": [2, 4], "financ": 2, "law": 2, "suitabl": 2, "serv": [2, 3, 4], "aproach": 2, "significantli": [2, 3], "workflow": [2, 4], "assessor": 2, "intens": [2, 4], "aplic": 2, "aim": [2, 3, 4], "clearli": [2, 4], "earlier": 2, "depict": [2, 4], "higher": 2, "correl": 2, "were": [2, 4], "multilingu": 2, "golden": 2, "recruit": 2, "languang": 2, "extens": [2, 3, 4], "arena": 2, "vote": 2, "made": [2, 3, 4], "under": [2, 4], "blind": 2, "randomli": 2, "pair": 2, "submit": 2, "actual": [2, 3, 4], "loop": 2, "customiz": 2, "irrelev": 2, "unhelp": 2, "sometim": [2, 4], "though": [2, 4], "occasion": 2, "regularli": 2, "inquiri": 2, "rare": 2, "inaccuraci": 2, "highli": [2, 4], "perfectli": 2, "cater": 2, "polici": 2, "benefit": [2, 4], "critiqu": 2, "elo": 2, "democrat": [2, 4], "simul": [2, 4], "thought": [2, 4], "exam": 2, "probe": 2, "certifi": 2, "identif": 2, "histori": 2, "prioriti": 2, "intellig": 2, "move": [2, 3], "began": 2, "2018": 2, "introduct": [2, 3], "glue": 2, "entail": 2, "baselin": 2, "superglu": 2, "2019": 2, "expand": 2, "deeper": [2, 3], "successor": 2, "grew": 2, "broader": 2, "big": 2, "bench": 2, "2021": 2, "turn": 2, "200": 2, "span": 2, "arithmet": 2, "collabor": 2, "truthfulqa": 2, "accur": [2, 4], "decept": 2, "increasingli": [2, 4], "stanford": 2, "helm": 2, "multidimension": 2, "concern": 2, "surround": [2, 4], "emphas": [2, 4], "multitask": 2, "multidisciplinari": 2, "57": 2, "humanev": 2, "lmsy": 2, "2023": 2, "brought": 2, "dialogu": 2, "len": [2, 3], "dai": [2, 4], "groundbreak": 2, "arc": 2, "prize": 2, "steadi": 2, "progress": [2, 3], "meaning": [2, 3, 4], "bar": 2, "ambit": 2, "so": [2, 4], "track": 2, "replic": [2, 4], "find": [2, 3, 4], "industri": [2, 4], "chatbot": 2, "formerli": 2, "direct": 2, "live": 2, "gather": 2, "000": [2, 4], "assist": [2, 4], "alpacaev": 2, "mt": 2, "Their": [2, 4], "came": 2, "center": [2, 4], "mike": 2, "knoop": 2, "co": 2, "founder": 2, "zapier": 2, "fran\u00e7oi": 2, "chollet": 2, "creator": 2, "agi": 2, "kera": 2, "narrow": 2, "suffici": [2, 4], "capac": 2, "genuin": 2, "accord": 2, "wrong": 2, "econom": 2, "acquir": 2, "skill": 2, "five": 2, "old": 2, "possess": 2, "count": [2, 3], "elementari": 2, "physic": 2, "novelti": 2, "puzzl": 2, "novel": 2, "someth": 2, "wouldn": 2, "vast": 2, "interpol": 2, "memori": [2, 3], "synthes": 2, "fly": 2, "brute": 2, "possibl": [2, 4], "million": 2, "seri": 2, "minim": [2, 4], "prior": 2, "submiss": 2, "pixel": 2, "perfect": 2, "color": 2, "modern": [2, 3, 4], "meaningfulli": 2, "fourrier": 2, "lightweight": [2, 4], "varieti": 2, "bespok": 2, "via": [2, 4], "sdk": 2, "cli": 2, "been": 2, "extract": [2, 3, 4], "autoregress": 2, "conduct": 2, "sub": 2, "liter": 2, "disturb": 2, "zero": 2, "varianc": 2, "yt": 2, "ut": 2, "uncondit": 2, "33": 2, "suppos": 2, "p": 2, "08": [2, 4], "exactli": [2, 4], "ii": [2, 4], "iv": 2, "iii": 2, "c": [2, 4], "consequ": 2, "ol": 2, "heteroscedast": 2, "regress": 2, "ineffici": 2, "wish": 2, "lag": 2, "var": 2, "bivari": 2, "acceler": 2, "evaluation_track": 2, "evaluationtrack": 2, "model_config": 2, "basemodelconfig": 2, "parallelismmanag": 2, "pipelineparamet": 2, "envconfig": 2, "is_accelerate_avail": 2, "datetim": 2, "timedelta": 2, "initprocessgroupkwarg": 2, "create_evaluation_pipelin": 2, "output_dir": 2, "cache_dir": 2, "pretrain": 2, "dtype": 2, "float16": 2, "max_sampl": 2, "kwargs_handl": 2, "second": [2, 3], "3000": 2, "els": [2, 3], "none": 2, "save_detail": 2, "push_to_hub": 2, "pipeline_param": 2, "launcher_typ": 2, "env_config": 2, "override_batch_s": 2, "use_chat_templ": 2, "trust_remote_cod": 2, "pipeline_paramet": 2, "schemat": [2, 3], "vllm": [2, 4], "tgi": 2, "instanti": 2, "storag": 2, "local": [2, 3, 4], "push": 2, "hub": 2, "parallel": 2, "temporari": 2, "maximum": [2, 3], "num_few_shot": 2, "automat": 2, "string": [2, 4], "vertic": 2, "few": [2, 3, 4], "binari": 2, "flag": 2, "bigbench": 2, "winogrand": 2, "hellaswag": 2, "nlp": 2, "choos": 2, "1b": 2, "save": [2, 3], "save_and_push_result": 2, "show_result": 2, "model_arg": 2, "download": 2, "remot": 2, "send": [2, 4], "serverless": 2, "dedic": [2, 4], "id": 2, "inference_server_address": 2, "inference_server_auth": 2, "model_id": 2, "null": 2, "bash": 2, "command": 2, "model_config_path": 2, "path": [2, 3], "endpoint_model": 2, "yaml": [2, 4], "llama3": [2, 3], "qwen2": [2, 4], "smollm2": 2, "describ": 2, "3b": 2, "alibaba": [2, 4], "5b": [2, 4], "hui": 2, "yang": 2, "compact": 2, "360m": 2, "allal": 2, "9": 2, "trend": [2, 4], "cluster": 2, "own": [2, 3], "degre": 2, "noteworthi": 2, "superior": 2, "taken": [2, 4], "grain": [2, 4], "salt": [2, 4], "100": [2, 4], "give": 2, "trade": [2, 4], "off": [2, 3, 4], "flexibl": [2, 3, 4], "exponenti": 2, "growth": 2, "hug": [2, 4], "ecosystem": 2, "modular": 2, "visit": 2, "offici": 2, "alb": 2, "24": [2, 4], "loubna": 2, "ben": 2, "anton": 2, "lozhkov": 2, "eli": 2, "bakouch": 2, "gabriel": 2, "mart\u00edn": 2, "bl\u00e1zquez": 2, "lewi": 2, "tunstal": 2, "agust\u00edn": 2, "piquer": 2, "andr": 2, "marafioti": 2, "cyril": 2, "zakka": 2, "leandro": 2, "von": 2, "werra": 2, "thoma": 2, "wolf": 2, "are24": 2, "judgearena": 2, "fac24a": 2, "wiki": [2, 4], "fac24b": 2, "fac24c": 2, "doc": [2, 4], "model_doc": 2, "gpt2": 2, "fac24d": 2, "cookbook": 2, "en": [2, 4], "llm_judg": 2, "fac24": 2, "fhwt23": 2, "cl\u00e9mentin": 2, "nathan": 2, "habib": 2, "url": [2, 4], "hyc": 2, "binyuan": 2, "jian": 2, "zeyu": 2, "cui": 2, "jiaxi": 2, "dayiheng": 2, "liu": [2, 4], "lei": 2, "zhang": 2, "tianyu": 2, "jiajun": 2, "bowen": 2, "yu": 2, "kai": 2, "dang": 2, "coder": 2, "arxiv": [2, 4], "preprint": [2, 4], "2409": 2, "12186": 2, "lx": 2, "zhen": 2, "xiaohan": 2, "xu": 2, "tao": 2, "shen": 2, "jia": 2, "chen": [2, 4], "gu": 2, "yuxuan": 2, "lai": 2, "chongyang": 2, "shuai": 2, "ma": 2, "nlg": 2, "org": [2, 4], "ab": [2, 4], "2401": 2, "07103": 2, "wtb": 2, "22": 2, "jason": 2, "yi": [2, 4], "tai": 2, "rishi": 2, "bommasani": 2, "colin": 2, "raffel": 2, "barret": 2, "zoph": 2, "sebastian": 2, "borgeaud": 2, "dani": 2, "yogatama": 2, "maarten": 2, "bosma": 2, "denni": 2, "zhou": 2, "donald": 2, "metzler": 2, "ed": 2, "h": 2, "chi": [2, 4], "tatsunori": 2, "hashimoto": 2, "oriol": 2, "vinyal": 2, "perci": 2, "liang": 2, "jeff": 2, "dean": 2, "william": 2, "fedu": 2, "2206": 2, "07682": 2, "yyh": 2, "baosong": 2, "bo": 2, "zheng": 2, "chengpeng": 2, "chengyuan": 2, "fei": 2, "huang": 2, "guant": 2, "dong": 2, "haoran": 2, "huan": 2, "lin": [2, 4], "jialong": 2, "tang": 2, "jialin": 2, "wang": 2, "jianhong": 2, "tu": 2, "jianwei": 2, "jianxin": 2, "jin": 2, "jingren": 2, "jinz": 2, "bai": 2, "jinzheng": 2, "junyang": 2, "keme": 2, "lu": 2, "keqin": 2, "kexin": 2, "mei": 2, "mingfeng": 2, "xue": 2, "na": 2, "ni": 2, "pei": 2, "peng": 2, "ru": 2, "rui": [2, 4], "men": 2, "ruiz": 2, "gao": 2, "runji": 2, "shiji": 2, "sinan": 2, "tan": 2, "tianhang": 2, "zhu": 2, "tianhao": 2, "wenbin": 2, "ge": 2, "xiaodong": 2, "deng": 2, "xiaohuan": 2, "xingzhang": 2, "ren": 2, "xinyu": 2, "xipin": 2, "xuancheng": 2, "fan": 2, "yao": 2, "yichang": 2, "wan": 2, "yunfei": 2, "chu": 2, "yuqiong": 2, "zhenru": 2, "zhihao": 2, "2407": 2, "10671": 2, "huggingface24": 2, "space": 2, "12": 2, "06": [2, 4], "metaai24": 2, "promptfoo24": 2, "toolkit": 2, "www": 2, "dev": 2, "go": [3, 4], "far": 3, "possibli": 3, "eliot": 3, "english": 3, "rule": [3, 4], "thumb": 3, "\u00be": 3, "max_output_token": 3, "4096": 3, "16384": 3, "contrari": 3, "surpass": 3, "stop": 3, "mid": 3, "truncat": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "11b": 3, "v1": 3, "128000": 3, "5e": 3, "sonnet": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "0613": 3, "6e": 3, "04": 3, "09": 3, "1e": 3, "gemini": 3, "flash": 3, "002": 3, "1048576": 3, "pro": 3, "2097152": 3, "05e": 3, "pose": [3, 4], "incomplet": 3, "articl": 3, "abruptli": 3, "cut": 3, "disrupt": 3, "shallow": 3, "thorough": 3, "receiv": 3, "partial": 3, "dissatisfact": 3, "frustrat": 3, "educ": 3, "creation": 3, "feasibl": 3, "split": 3, "previou": [3, 4], "10k": 3, "diagram": 3, "charactertextsplitt": 3, "tiktoken": 3, "sequenti": 3, "chain": 3, "newlin": 3, "broadli": [3, 4], "decid": 3, "want": 3, "sure": [3, 4], "lost": 3, "cheap": 3, "speciali": 3, "advantag": [3, 4], "naiv": 3, "period": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "manner": [3, 4], "talk": 3, "theme": 3, "topic": [3, 4], "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "persona": 3, "assum": 3, "task": [3, 4], "action": 3, "langchain_cor": [3, 4], "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": [3, 4], "from_templ": 3, "llmchain": 3, "togeth": 3, "parser": [3, 4], "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": [3, 4], "llm_chain": [3, 4], "api_key_label": 3, "upper": 3, "_api_kei": 3, "api_kei": 3, "get_dynamic_prompt_templ": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "origin": [3, 4], "part": [3, 4], "total": [3, 4], "param": 3, "dynamic_prompt_param": 3, "copi": 3, "elif": 3, "last": [3, 4], "merg": 3, "concaten": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "dinam": 3, "priovid": 3, "enumer": 3, "invok": [3, 4], "cummul": 3, "join": 3, "max_chunk_s": 3, "max_chunk_overlap": 3, "latest": [3, 4], "readabl": 3, "apple_report": 3, "300": 3, "disclos": [3, 4], "state": [3, 4], "luation": 3, "oblig": 3, "cash": 3, "disciplin": 3, "smooth": 3, "upon": 3, "subhead": 3, "adher": [3, 4], "revenu": [3, 4], "segment": [3, 4], "liquid": 3, "capit": [3, 4], "despit": [3, 4], "depth": 3, "overlook": 3, "mitig": [3, 4], "fit": [3, 4], "within": [3, 4], "preserv": 3, "easier": [3, 4], "preprocess": 3, "enhanc": [3, 4], "necessit": 3, "meticul": 3, "retain": 3, "necessari": 3, "seamlessli": 3, "circumv": 3, "therebi": 3, "escal": 3, "frequenc": 3, "volum": 3, "bottleneck": 3, "latenc": 3, "friendli": 3, "mustafa": 3, "suleyman": 3, "infinit": 3, "amount": [3, 4], "fewer": 3, "compress": 3, "condens": 3, "adjust": [3, 4], "constrain": [3, 4], "collect": 3, "versatil": 3, "drive": [3, 4], "grace": 3, "fallback": 3, "empow": 3, "crucial": [3, 4], "stai": 3, "full": [3, 4], "splitter": 3, "freedom": 4, "thrive": 4, "julia": 4, "cameron": 4, "easili": 4, "notebook": 4, "overrid": 4, "response_cont": 4, "wow": 4, "lot": 4, "breakdown": 4, "stream": 4, "portfolio": 4, "impress": 4, "notic": 4, "march": 4, "29": 4, "huge": 4, "investor": 4, "figur": 4, "compli": 4, "ye": 4, "date": 4, "serious": 4, "is_json": 4, "myjson": 4, "except": 4, "valueerror": 4, "lack": 4, "googl": 4, "survei": 4, "51": 4, "trial": 4, "elicit": 4, "consum": 4, "wrangl": 4, "conform": 4, "ad": 4, "hoc": 4, "streamlin": 4, "modul": 4, "dataset": 4, "unwant": 4, "neg": 4, "ui": 4, "restrict": 4, "mobil": 4, "devic": 4, "overflow": 4, "overwhelm": 4, "twitter": 4, "youtub": 4, "impos": 4, "publish": 4, "successfulli": 4, "adopt": 4, "emploi": 4, "schema": 4, "blueprint": 4, "nativ": 4, "regular": 4, "json_format": 4, "person1": 4, "alic": 4, "q1": 4, "20": 4, "person2": 4, "bob": 4, "net": 4, "margin": 4, "materi": 4, "nest": 4, "todai": 4, "programmat": 4, "thellm": 4, "unend": 4, "whitespac": 4, "until": 4, "forget": 4, "throw": 4, "appear": 4, "somewher": 4, "json_object": 4, "628": 4, "553": 4, "sheer": 4, "115": 4, "823": 4, "circul": 4, "plai": 4, "vertex": 4, "releas": 4, "suppli": 4, "worri": 4, "enum": 4, "No": 4, "incorrectli": 4, "refus": 4, "simpler": 4, "strongli": 4, "entiti": 4, "place": 4, "07": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "hint": 4, "attribut": 4, "prompt_extract": 4, "sec_extract": 4, "nasdaq": 4, "llc": 4, "washington": 4, "cupertino": 4, "usabl": 4, "beg": 4, "with_structured_output": 4, "runnabl": 4, "typeddict": 4, "qu": 4, "langchain_openai": 4, "chatopenai": 4, "chatprompttempl": 4, "extract_from_sec_filing_langchain": 4, "structured_llm": 4, "from_messag": 4, "sec_extraction_langchain": 4, "found": 4, "hood": 4, "logit": 4, "raw": 4, "neural": 4, "network": 4, "regex": 4, "strong": 4, "enough": 4, "qwen": 4, "top": 4, "label": 4, "unexpect": 4, "malform": 4, "pass": 4, "sec_extraction_outlin": 4, "zsp": 4, "zicorp": 4, "phenomenon": 4, "popular": 4, "cpp": 4, "gbnf": 4, "ggml": 4, "bnf": 4, "ggerganov": 4, "accomplish": 4, "formal": 4, "backu": 4, "naur": 4, "wikipedia": 4, "contributor": 4, "strictli": 4, "soon": 4, "curl": 4, "fssl": 4, "sh": 4, "did": 4, "extract_entities_from_sec_fil": 4, "suffix": 4, "ollama_structured_output_prompt_suffix": 4, "ollama_structured_output_temperatur": 4, "mistral": 4, "llama2": 4, "uncensor": 4, "model_json_schema": 4, "response_json": 4, "AND": 4, "wrapper": 4, "exllama2": 4, "mlx": 4, "lm": 4, "enterpris": 4, "commerci": 4, "medium": 4, "low": 4, "done": 4, "know": 4, "chanc": 4, "connect": 4, "correctli": 4, "area": 4, "mix": 4, "famili": 4, "furthermor": 4, "nonetheless": 4, "evid": 4, "studi": 4, "wrap": 4, "map": 4, "gemma": 4, "uncov": 4, "wors": 4, "extran": 4, "dispar": 4, "preval": 4, "outdat": 4, "rapidli": 4, "fashion": 4, "remark": 4, "me": 4, "speak": 4, "freeli": 4, "tam": 4, "aider": 4, "decod": 4, "hinder": 4, "outweigh": 4, "team": 4, "rebutt": 4, "argu": 4, "v": 4, "compel": 4, "reproduct": 4, "paint": 4, "pictur": 4, "publicli": 4, "independ": 4, "verif": 4, "dottxt": 4, "flaw": 4, "believ": 4, "led": 4, "inaccur": 4, "reconcil": 4, "uneven": 4, "didn": 4, "conflat": 4, "argument": 4, "drawback": 4, "unlock": 4, "wider": 4, "thank": 4, "pfiffer": 4, "hi": 4, "aid24": 4, "html": 4, "dot24": 4, "sai": 4, "demo": 4, "tree": 4, "gge24": 4, "blob": 4, "readm": 4, "md": 4, "llf": 4, "michael": 4, "xieyang": 4, "frederick": 4, "alexand": 4, "j": 4, "fiannaca": 4, "terri": 4, "koo": 4, "luca": 4, "dixon": 4, "carri": 4, "cai": 4, "ea": 4, "york": 4, "ny": 4, "usa": 4, "machineri": 4, "doi": 4, "1145": 4, "3613905": 4, "3650756": 4, "ln": 4, "xuan": 4, "hai": 4, "nguyen": 4, "ngoc": 4, "tiviati": 4, "sim": 4, "hieu": 4, "dao": 4, "shafiq": 4, "joti": 4, "kenji": 4, "kawaguchi": 4, "nanci": 4, "min": 4, "yen": 4, "kan": 4, "2408": 4, "08656": 4, "out24": 4, "io": 4, "twt": 4, "zhi": 4, "cheng": 4, "kuang": 4, "wu": 4, "tsai": 4, "chieh": 4, "hung": 4, "lee": 4, "yun": 4, "nung": 4, "02442": 4, "wikipediacontributors24": 4, "wiktionari": 4, "naur_form": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1, 4], "content": [0, 2, 3, 4], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 1], "practic": [0, 1, 4], "approach": 0, "note": 0, "perspect": 0, "who": 0, "thi": 0, "book": 0, "i": 0, "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "python": 0, "setup": 0, "api": [0, 4], "kei": [0, 2, 3], "configur": 0, "code": 0, "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "about": 0, "author": 0, "": 0, "tame": 1, "llm": [1, 2], "guid": 1, "pitfal": 1, "open": 1, "sourc": 1, "softwar": [1, 2], "chapter": 1, "1": [1, 3], "2": [1, 3], "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "3": [1, 3], "input": 1, "size": [1, 3], "length": [1, 3], "limit": [1, 3], "4": [1, 3], "5": 1, "The": [1, 2], "eval": [1, 2], "gap": [1, 2], "6": 1, "hallucin": 1, "realiti": 1, "7": 1, "safeti": 1, "concern": 1, "8": 1, "cost": [1, 3], "factor": 1, "9": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "tool": [1, 2, 4], "resourc": 1, "non": 2, "determinist": 2, "machin": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "emerg": 2, "properti": 2, "problem": [2, 3, 4], "statement": [2, 3, 4], "tradit": 2, "v": 2, "design": 2, "applic": 2, "test": 2, "requir": 2, "matrix": 2, "conceptu": 2, "overview": 2, "consider": [2, 3], "metric": 2, "evalu": 2, "gener": [2, 3], "task": 2, "model": [2, 3], "base": [2, 3], "human": 2, "benchmark": 2, "leaderboard": 2, "lightev": 2, "mmlu": 2, "econometr": 2, "dataset": 2, "famili": 2, "us": 2, "langchain": [2, 4], "promptfoo": 2, "refer": [2, 3, 4], "what": 3, "ar": 3, "token": 3, "comparison": [3, 4], "across": 3, "chunk": 3, "contextu": 3, "link": 3, "long": 3, "form": 3, "step": 3, "write": 3, "prompt": [3, 4], "templat": 3, "construct": 3, "dynam": 3, "paramet": 3, "report": 3, "exampl": 3, "usag": 3, "discuss": [3, 4], "implic": 3, "futur": 3, "conclus": [3, 4], "user": 4, "need": 4, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "specif": 4, "json": 4, "mode": 4, "outlin": 4, "ollama": 4, "compar": 4, "framework": 4, "best": 4, "research": 4, "ongo": 4, "debat": 4, "acknowledg": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"], [4, "introduction"]], "Contents": [[0, "contents"], [2, "contents"], [3, "contents"], [4, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author(s)": [[0, "about-the-author-s"]], "Taming LLMs": [[1, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[1, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Wrestling with Structured Output": [[1, "chapter-2-wrestling-with-structured-output"]], "Chapter 3: Input Size and Length Limitations": [[1, "chapter-3-input-size-and-length-limitations"]], "Chapter 4: Output Size and Length Limitations": [[1, "chapter-4-output-size-and-length-limitations"]], "Chapter 5: The Evals Gap": [[1, "chapter-5-the-evals-gap"]], "Chapter 6: Hallucination: The Reality Gap": [[1, "chapter-6-hallucination-the-reality-gap"]], "Chapter 7: Safety Concerns": [[1, "chapter-7-safety-concerns"]], "Chapter 8: The Cost Factor": [[1, "chapter-8-the-cost-factor"]], "Chapter 9: Breaking Free from Cloud Providers": [[1, "chapter-9-breaking-free-from-cloud-providers"]], "Appendix A: Tools and Resources": [[1, "appendix-a-tools-and-resources"]], "The Evals Gap": [[2, "the-evals-gap"]], "Non-Deterministic Machines": [[2, "non-deterministic-machines"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Emerging Properties": [[2, "emerging-properties"]], "Problem Statement": [[2, "problem-statement"], [3, "problem-statement"], [4, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[2, "evals-table"]], "Evals Design": [[2, "evals-design"]], "LLM Application Testing Requirements Matrix": [[2, "validation-requirements"]], "Conceptual Overview": [[2, "conceptual-overview"]], "Design Considerations": [[2, "design-considerations"]], "Metrics": [[2, "metrics"]], "Key Metrics for Evaluating Generative Tasks": [[2, "key-metrics"]], "Evaluators": [[2, "evaluators"]], "Model-Based Evaluation": [[2, "model-based-evaluation"]], "Human-Based Evaluation": [[2, "human-based-evaluation"]], "Evaluating Evaluators": [[2, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[2, "benchmarks-and-leaderboards"]], "Tools": [[2, "tools"]], "LightEval": [[2, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[2, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[2, "model-families"]], "LangChain": [[2, "langchain"], [4, "langchain"]], "PromptFoo": [[2, "promptfoo"]], "References": [[2, "references"], [3, "references"], [4, "references"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Token Cost and Length Limitation Comparison Across Key Models": [[3, "token-cost-table"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Step 2: Writing the Base Prompt Template": [[3, "step-2-writing-the-base-prompt-template"]], "Step 3: Constructing Dynamic Prompt Parameters": [[3, "step-3-constructing-dynamic-prompt-parameters"]], "Step 4: Generating the Report": [[3, "step-4-generating-the-report"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"], [4, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "User Needs": [[4, "user-needs"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "JSON Mode": [[4, "json-mode"]], "Outlines": [[4, "outlines"]], "Ollama": [[4, "ollama"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[4, "structured-output-frameworks"]], "Best Practices": [[4, "best-practices"]], "Research and Ongoing Debate": [[4, "research-and-ongoing-debate"]], "Acknowledgements": [[4, "acknowledgements"]]}, "indexentries": {}}) \ No newline at end of file +Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/evals", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/evals.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["1. Introduction", "Taming LLMs", "4. The Evals Gap", "2. Output Size Limitations", "3. Wrestling with Structured Output"], "terms": {"am": 0, "alwai": [0, 2, 4], "do": [0, 2, 3, 4], "which": [0, 2, 3, 4], "cannot": [0, 2], "order": [0, 2, 4], "mai": [0, 2, 3, 4], "learn": [0, 2], "how": [0, 2, 3, 4], "pablo": 0, "picasso": 0, "In": [0, 2, 3, 4], "recent": [0, 4], "year": [0, 2, 3, 4], "larg": [0, 1, 2, 3, 4], "languag": [0, 1, 2, 3, 4], "model": [0, 1, 4], "llm": [0, 3, 4], "have": [0, 2, 3, 4], "emerg": [0, 1, 4], "transform": [0, 2, 4], "forc": [0, 2, 4], "technologi": [0, 2, 3, 4], "promis": 0, "revolution": 0, "build": [0, 1, 2, 3, 4], "product": [0, 1, 2, 4], "interact": [0, 2, 3, 4], "comput": [0, 2, 3, 4], "from": [0, 2, 3, 4], "chatgpt": [0, 2, 4], "github": [0, 2, 4], "copilot": 0, "claud": [0, 2, 3], "artifact": 0, "system": [0, 2, 3, 4], "captur": [0, 2], "public": [0, 2], "imagin": 0, "spark": 0, "gold": [0, 2], "rush": 0, "ai": [0, 2, 4], "power": [0, 1, 2, 3, 4], "applic": [0, 1, 3, 4], "howev": [0, 2, 3, 4], "beneath": 0, "surfac": [0, 2], "technolog": 0, "revolut": 0, "li": [0, 2], "complex": [0, 2, 3, 4], "landscap": [0, 2], "practition": [0, 2], "must": [0, 2, 3], "navig": [0, 1], "focus": [0, 2, 3, 4], "bring": 0, "awar": [0, 2, 3], "limit": [0, 2, 4], "har": [0, 1, 3], "open": [0, 2, 3, 4], "sourc": [0, 2, 4], "solut": [0, 1, 2, 3], "overcom": [0, 3], "them": [0, 2, 3, 4], "robust": [0, 2, 3, 4], "It": [0, 2, 3, 4], "offer": [0, 2, 3, 4], "critic": [0, 1, 2, 3, 4], "implement": [0, 1, 2, 3, 4], "back": [0, 4], "reproduc": [0, 1, 2], "exampl": [0, 1, 2, 4], "while": [0, 1, 2, 3, 4], "mani": [0, 3, 4], "resourc": [0, 2, 3], "cover": [0, 2, 3], "capabl": [0, 1, 2, 3, 4], "specif": [0, 1, 2, 3], "hidden": 0, "pitfal": 0, "engin": [0, 1, 2, 4], "technic": [0, 1, 2, 3, 4], "manag": [0, 1, 2, 3], "face": [0, 2, 4], "when": [0, 1, 2, 3, 4], "comprehens": [0, 1, 2, 3, 4], "guid": [0, 2, 4], "leverag": [0, 2, 3, 4], "battl": [0, 1], "test": [0, 1, 4], "tool": [0, 3], "throughout": [0, 3, 4], "tackl": [0, 2], "follow": [0, 2, 3, 4], "non": [0, 1, 4], "exhaust": 0, "list": [0, 2, 3, 4], "structur": [0, 2, 3], "un": 0, "reliabl": [0, 2, 4], "struggl": [0, 2, 4], "maintain": [0, 2, 3, 4], "consist": [0, 2, 3, 4], "output": [0, 2], "format": [0, 2, 3, 4], "complic": 0, "integr": [0, 2, 4], "larger": [0, 2, 3, 4], "make": [0, 2, 3, 4], "error": [0, 2, 4], "handl": [0, 1, 2, 3, 4], "more": [0, 2, 3, 4], "size": [0, 2, 4], "length": [0, 2, 4], "constraint": [0, 1, 3, 4], "strict": [0, 4], "token": [0, 1, 2, 4], "both": [0, 2], "input": [0, 2, 3, 4], "requir": [0, 3, 4], "care": [0, 2, 4], "chunk": [0, 1], "strategi": [0, 1, 2, 3], "long": [0, 1, 2, 4], "form": [0, 1, 2, 4], "effect": [0, 2, 3, 4], "tradit": 0, "softwar": [0, 4], "methodologi": [0, 2, 4], "break": [0, 2, 3], "down": [0, 2, 3], "deal": 0, "determinist": [0, 1, 4], "gener": [0, 1, 4], "new": [0, 2, 3, 4], "hallucin": [0, 2, 4], "These": [0, 2, 3, 4], "can": [0, 2, 3, 4], "plausibl": 0, "sound": 0, "entir": [0, 2, 3], "fabric": [0, 2], "inform": [0, 2, 3, 4], "creat": [0, 2, 3, 4], "signific": [0, 2, 3, 4], "risk": [0, 2, 3], "safeti": [0, 2, 4], "secur": [0, 2, 3, 4], "harm": [0, 2], "bias": [0, 2, 4], "inappropri": 0, "safeguard": [0, 2], "monitor": [0, 1], "ensur": [0, 2, 3, 4], "safe": [0, 4], "deploy": [0, 1, 2, 4], "cost": [0, 2, 4], "optim": [0, 1, 2, 3], "The": [0, 3, 4], "financi": [0, 2, 3, 4], "oper": [0, 2, 3], "base": [0, 1, 4], "quickli": [0, 3], "becom": [0, 2], "prohibit": 0, "without": [0, 2, 3, 4], "observ": [0, 2, 4], "vendor": [0, 1, 2], "lock": [0, 1], "cloud": [0, 2, 4], "provid": [0, 2, 3], "depend": [0, 2, 4], "through": [0, 1, 2, 3, 4], "proprietari": [0, 4], "infrastructur": 0, "difficult": [0, 2], "switch": 0, "self": [0, 1, 2], "host": [0, 1, 2], "take": [0, 1, 2, 3, 4], "hand": [0, 3, 4], "concret": [0, 1], "you": [0, 2, 3, 4], "run": [0, 2, 4], "modifi": 0, "real": [0, 2, 3, 4], "world": [0, 2, 4], "scenario": [0, 2, 4], "best": [0, 1, 2], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 2, 4], "anti": 0, "look": [0, 1, 2], "our": [0, 2, 3, 4], "goal": [0, 2, 3], "discourag": 0, "us": [0, 3, 4], "enabl": [0, 2, 3, 4], "By": [0, 1, 2, 3, 4], "understand": [0, 1, 2, 3, 4], "upfront": [0, 1], "better": [0, 1, 2, 3], "equip": [0, 1], "avoid": [0, 2, 4], "current": [0, 1, 3, 4], "discours": [0, 1], "around": [0, 1, 2, 3, 4], "tend": [0, 1, 2], "toward": [0, 2, 4], "extrem": 0, "either": [0, 2, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": 0, "dismiss": 0, "differ": [0, 2, 3, 4], "focu": [0, 1, 2, 3, 4], "rather": [0, 2], "than": [0, 2], "theoret": 0, "examin": [0, 3, 4], "first": [0, 2, 3, 4], "everi": 0, "concept": [0, 2], "illustr": [0, 2, 3], "execut": [0, 2], "immedi": 0, "analysi": [0, 1, 2, 3], "balanc": [0, 2, 3, 4], "help": [0, 2, 3, 4], "reader": [0, 1], "decis": [0, 2, 4], "intend": 0, "develop": [0, 2, 3, 4], "step": [0, 1, 2, 4], "insight": [0, 2, 3, 4], "along": [0, 2], "guidanc": [0, 4], "framework": [0, 2], "could": [0, 2, 3, 4], "derail": 0, "project": 0, "earli": [0, 2, 4], "befor": [0, 2, 4], "thei": [0, 2, 3, 4], "costli": 0, "problem": [0, 1], "too": [0, 2, 3], "late": 0, "lifecycl": 0, "design": [0, 1, 3, 4], "lead": [0, 2, 3, 4], "genai": 0, "initi": [0, 2, 3], "leader": 0, "architectur": [0, 2, 3], "advoc": 0, "anyon": 0, "seek": [0, 2], "work": [0, 1, 2, 3, 4], "typic": [0, 2, 3], "job": 0, "role": [0, 2, 3, 4], "platform": [0, 2, 3, 4], "backend": [0, 2], "exist": [0, 2], "ml": 0, "transit": [0, 2, 3], "overse": 0, "motiv": [0, 2, 4], "need": [0, 2, 3], "readi": [0, 2], "desir": [0, 2, 4], "perform": [0, 1, 2, 3, 4], "after": [0, 2, 3], "read": [0, 2, 3, 4], "implic": [0, 1, 2], "experi": [0, 2, 3, 4], "recommend": [0, 2, 3, 4], "abl": [0, 3, 4], "deploi": [0, 3], "proper": [0, 4], "realist": 0, "effort": [0, 2, 4], "estim": [0, 2], "impact": [0, 2, 3, 4], "timelin": 0, "To": [0, 2, 3, 4], "most": [0, 2, 3, 4], "should": [0, 2, 3, 4], "basic": [0, 2, 3], "program": [0, 2], "knowledg": [0, 2], "introductori": [0, 1], "langchain": [0, 1, 3], "e": [0, 2, 3, 4], "g": [0, 2, 3, 4], "chat": [0, 2, 3, 4], "prompt": [0, 1, 2], "templat": [0, 1, 2], "access": [0, 2, 3, 4], "openai": [0, 2, 4], "anthrop": [0, 4], "similar": [0, 2, 4], "grade": 0, "dive": 0, "here": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": [0, 2, 4], "activ": [0, 2], "virtual": [0, 2], "m": 0, "venv": 0, "env": [0, 2, 3, 4], "bin": 0, "On": [0, 4], "window": [0, 1], "script": 0, "instal": [0, 2, 4], "packag": 0, "pip": [0, 2, 4], "r": [0, 2, 3, 4], "txt": [0, 2, 3, 4], "file": [0, 2, 3, 4], "root": 0, "directori": [0, 2], "add": [0, 3], "other": [0, 2, 3, 4], "sensit": [0, 2], "openai_api_kei": 0, "your_openai_api_key_her": 0, "never": 0, "share": [0, 2, 4], "commit": [0, 2], "version": [0, 2, 4], "control": [0, 2, 4], "contain": [0, 2, 3, 4], "kept": [0, 2], "privat": 0, "clone": 0, "companion": 0, "git": 0, "http": [0, 2, 3, 4], "com": [0, 2, 3, 4], "souzatharsi": 0, "tamingllm": 0, "cd": 0, "If": [0, 2, 4], "encount": [0, 1, 2], "rate": [0, 2], "consid": [0, 2, 3, 4], "smaller": [0, 2, 3, 4], "retri": [0, 4], "logic": [0, 2, 3], "conflict": 0, "try": [0, 2, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": [0, 2], "page": 0, "known": [0, 2, 4], "now": [0, 2, 3, 4], "let": [0, 2, 3, 4], "begin": [0, 2], "explor": [0, 2, 4], "dr": 0, "tharsi": 0, "souza": 0, "scientist": 0, "special": [0, 2, 4], "he": [0, 2], "lectur": 0, "columbia": 0, "univers": 0, "master": [0, 4], "scienc": [0, 2], "appli": [0, 2, 3], "analyt": 0, "head": [0, 3], "equiti": 0, "citadel": 0, "former": [0, 2], "senior": 0, "vp": 0, "two": [0, 2, 3, 4], "sigma": 0, "invest": [0, 2, 4], "With": 0, "over": [0, 1, 2, 3, 4], "15": [0, 2, 4], "deliv": 0, "across": [0, 2, 4], "startup": 0, "fortun": 0, "500": [0, 2], "compani": [0, 2, 3, 4], "global": 0, "also": [0, 2, 3, 4], "an": [0, 1, 2, 3, 4], "numer": [0, 2], "scholarli": 0, "frequent": [0, 4], "speaker": 0, "academ": [0, 2], "busi": [0, 2], "confer": [0, 4], "ground": [0, 1, 2], "background": [0, 3], "draw": [0, 4], "scale": [0, 2, 4], "stage": 0, "major": [0, 2, 4], "institut": 0, "well": [0, 2, 4], "advis": 0, "profit": [0, 2, 3, 4], "organ": [0, 2, 3], "contribut": [0, 2, 3], "uniqu": [0, 2], "bridg": 0, "gap": 0, "between": [0, 2, 3, 4], "potenti": [0, 2, 3, 4], "next": [0, 2, 4], "hold": 0, "ph": 0, "d": [0, 4], "ucl": 0, "london": 0, "phil": 0, "sc": 0, "b": [0, 2, 4], "abstract": [1, 2, 4], "heavili": [1, 2, 4], "gloss": 1, "fundament": [1, 2, 4], "challeng": [1, 2, 3, 4], "convers": [1, 2, 3, 4], "thi": [1, 2, 3, 4], "book": 1, "kei": [1, 4], "python": [1, 2, 3, 4], "proven": 1, "yet": [1, 2, 3], "i": [1, 2, 3, 4], "unstructur": [1, 4], "context": [1, 2, 3, 4], "code": [1, 2, 4], "sidestep": 1, "inher": [1, 2, 3, 4], "core": [1, 2], "we": [1, 2, 3, 4], "ll": [1, 2], "address": [1, 2, 3, 4], "approach": [1, 2, 3, 4], "note": [1, 2, 3, 4], "perspect": 1, "who": [1, 2, 3, 4], "For": [1, 2, 3, 4], "outcom": [1, 2, 4], "prerequisit": 1, "set": [1, 2, 3, 4], "up": [1, 2, 3, 4], "your": [1, 2, 3, 4], "environ": [1, 2, 3, 4], "setup": [1, 2, 4], "api": [1, 2], "configur": [1, 2], "repositori": [1, 2], "troubleshoot": 1, "common": [1, 2, 3, 4], "issu": [1, 2, 3, 4], "about": [1, 2, 3, 4], "author": [1, 4], "": [1, 2, 3, 4], "statement": 1, "One": [1, 2], "shot": [1, 2], "json": [1, 2, 3], "mode": 1, "outlin": [1, 2], "multipl": [1, 2, 3, 4], "choic": [1, 2, 4], "pydant": [1, 2, 4], "discuss": [1, 2], "compar": [1, 2, 3], "research": [1, 2, 3], "ongo": [1, 2], "debat": 1, "conclus": [1, 2], "acknowledg": [1, 2], "refer": 1, "content": 1, "what": [1, 2, 4], "ar": [1, 2, 4], "contextu": [1, 2], "link": 1, "write": [1, 2, 4], "construct": [1, 2, 4], "dynam": [1, 2], "paramet": [1, 2, 4], "report": [1, 2, 4], "usag": [1, 2, 4], "futur": [1, 2], "consider": [1, 4], "machin": 1, "temperatur": [1, 3, 4], "sampl": [1, 3, 4], "spectrum": 1, "properti": 1, "conceptu": [1, 4], "overview": [1, 4], "compon": [1, 2], "metric": 1, "evalu": [1, 3, 4], "human": [1, 3, 4], "benchmark": 1, "leaderboard": 1, "type": [1, 2, 3, 4], "detect": [1, 2, 4], "retriev": [1, 2], "augment": [1, 2], "rag": 1, "select": [1, 2], "index": [1, 2, 3], "vector": 1, "store": [1, 2, 3], "method": [1, 2, 3, 4], "pipelin": [1, 2, 4], "valid": [1, 2, 4], "guard": 1, "filter": [1, 2], "sanit": 1, "alert": 1, "cach": [1, 2], "invalid": [1, 4], "predict": [1, 2, 4], "llama": [1, 2, 4], "llamafil": 1, "ollama": 1, "migrat": 1, "commun": [1, 2, 4], "surprisingli": 2, "often": [2, 3, 4], "all": [2, 3, 4], "greg": 2, "brockman": 2, "presid": 2, "natur": [2, 3, 4], "unlik": 2, "where": [2, 3, 4], "same": [2, 3, 4], "produc": [2, 4], "respons": [2, 3, 4], "each": [2, 3], "time": [2, 3, 4], "re": [2, 3, 4], "queri": 2, "even": [2, 3, 4], "ident": 2, "data": [2, 3, 4], "characterist": 2, "strength": 2, "ask": [2, 4], "ani": [2, 3, 4], "question": [2, 4], "isn": 2, "t": [2, 3, 4], "bug": 2, "featur": [2, 4], "random": 2, "allow": [2, 3, 4], "creativ": [2, 4], "divers": [2, 3, 4], "incredibli": 2, "testabl": 2, "servic": [2, 3, 4], "advic": 2, "mean": [2, 3, 4], "market": [2, 3, 4], "yield": 2, "exceedingli": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3], "trust": [2, 4], "affect": 2, "inconsist": [2, 4], "primari": 2, "determin": [2, 3, 4], "come": [2, 3, 4], "dure": [2, 4], "text": [2, 3, 4], "calcul": 2, "probabl": [2, 4], "distribut": [2, 4], "nucleu": 2, "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3, 4], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "dotenv": [2, 3, 4], "import": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "variabl": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 4], "datafram": 2, "demonstr": [2, 3, 4], "behavior": 2, "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3, 4], "complet": [2, 3, 4], "messag": [2, 4], "max_token": 2, "50": 2, "append": [2, 3, 4], "displai": [2, 4], "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": 2, "row": 2, "iterrow": 2, "return": [2, 3, 4], "max_length": [2, 4], "10000": [2, 3, 4], "appl": [2, 3, 4], "sec_fil": [2, 4], "gpt": [2, 3, 4], "5": [2, 3, 4], "turbo": [2, 3, 4], "singl": [2, 3, 4], "summari": [2, 4], "2": [2, 4], "inc": [2, 3, 4], "its": [2, 3, 4], "10": [2, 3, 4], "k": [2, 3, 4], "fiscal": [2, 3], "end": [2, 3], "septemb": [2, 3], "28": [2, 3], "2024": [2, 3, 4], "sec": [2, 3, 4], "detail": [2, 3, 4], "season": 2, "issuer": 2, "california": [2, 4], "manufactur": 2, "smartphon": 2, "person": [2, 4], "tablet": 2, "wearabl": [2, 4], "accessori": 2, "innov": [2, 3], "condit": 2, "exchang": [2, 3, 4], "commiss": [2, 3, 4], "factor": [2, 3, 4], "invdestacksmeticsisdict": 2, "setispect": 2, "20cyan": 2, "evaluationseld": 2, "anvis": 2, "droitent": 2, "discernminerv": 2, "versbobprefvers": 2, "vo\u8be5": 2, "option\u548c": 2, "meio": 2, "forecast": 2, "\u0432\u0440\u0435\u043ccisco": 2, "dellaischenpoihscap": 2, "geme": 2, "gettim": 2, "simpl": [2, 3, 4], "reveal": 2, "dramat": [2, 4], "alter": 2, "wai": [2, 3, 4], "systemat": [2, 4], "assess": [2, 3], "At": 2, "rigid": 2, "vari": 2, "less": 2, "wildli": 2, "inadequ": 2, "profound": 2, "one": [2, 3, 4], "radic": 2, "reli": 2, "u": [2, 4], "grappl": 2, "probabilist": 2, "lower": [2, 4], "seem": [2, 4], "safer": 2, "don": [2, 3, 4], "elimin": 2, "underli": [2, 4], "uncertainti": 2, "mere": 2, "mask": 2, "highlight": [2, 3, 4], "paradigm": 2, "aspect": [2, 3, 4], "beyond": 2, "present": [2, 3, 4], "anoth": 2, "fascin": 2, "abil": [2, 4], "spontan": 2, "aris": 2, "answer": [2, 3, 4], "reason": [2, 3, 4], "aren": 2, "explicitli": 2, "grow": [2, 4], "train": [2, 4], "against": 2, "clear": [2, 4], "wei": 2, "et": [2, 4], "al": [2, 4], "2022": 2, "fig": [2, 3, 4], "4": 2, "relationship": 2, "linear": 2, "below": [2, 3, 4], "certain": [2, 3, 4], "threshold": 2, "absent": 2, "simpli": [2, 3, 4], "matter": 2, "much": 2, "coax": 2, "out": [2, 3], "onc": [2, 3], "reach": [2, 3, 4], "point": [2, 3], "journei": 2, "suddenli": 2, "manifest": 2, "call": [2, 3, 4], "phase": 2, "shift": 2, "inabl": 2, "unpredict": [2, 4], "stand": 2, "stark": 2, "contrast": 2, "deliber": 2, "convent": 2, "stabl": 2, "suit": 2, "defin": [2, 3, 4], "accept": 2, "criteria": 2, "contend": 2, "constantli": 2, "7b": 2, "70b": 2, "ha": [2, 4], "rethink": 2, "practic": [2, 3], "math": 2, "tutor": 2, "children": 2, "would": [2, 3, 4], "verifi": [2, 4], "function": [2, 3, 4], "But": [2, 4], "just": [2, 3, 4], "predefin": [2, 4], "adapt": [2, 3], "explan": [2, 4], "child": 2, "level": [2, 3, 4], "engag": [2, 4], "ag": 2, "appropri": [2, 3, 4], "encourag": [2, 4], "emot": 2, "support": [2, 4], "rais": [2, 3], "measur": 2, "weren": 2, "evolv": [2, 3], "accuraci": [2, 4], "subject": 2, "qualiti": [2, 3, 4], "kind": 2, "There": [2, 3, 4], "account": 2, "tabl": [2, 3, 4], "sever": [2, 3, 4], "dimens": 2, "necessarili": 2, "pre": 2, "extend": [2, 4], "explicit": [2, 4], "usual": 2, "precis": [2, 4], "involv": [2, 4], "resist": 2, "straightforward": [2, 3, 4], "quantif": 2, "score": [2, 4], "judgment": 2, "remain": [2, 3], "contamin": 2, "carefulli": [2, 4], "craft": [2, 4], "case": [2, 3, 4], "expect": [2, 3, 4], "unit": [2, 3, 4], "massiv": 2, "internet": 2, "alreadi": 2, "seen": 2, "memor": 2, "artifici": 2, "inflat": 2, "curat": 2, "truli": 2, "unseen": 2, "rigor": 2, "cross": 2, "evolut": 2, "continu": [2, 3, 4], "advanc": [2, 3, 4], "longitudin": 2, "comparison": 2, "obsolet": 2, "older": 2, "autom": [2, 4], "demand": [2, 4], "oversight": 2, "annot": 2, "review": [2, 4], "process": [2, 3, 4], "mostli": [2, 4], "distinct": 2, "versu": 2, "latter": 2, "foundat": [2, 3], "purpos": [2, 4], "tailor": 2, "particular": [2, 4], "combin": [2, 3, 4], "associ": [2, 3, 4], "solv": [2, 4], "That": [2, 4], "differenti": 2, "becaus": 2, "chang": 2, "scope": [2, 3], "includ": [2, 3, 4], "thing": [2, 4], "instead": [2, 3, 4], "meet": [2, 4], "align": [2, 3, 4], "object": [2, 4], "A": [2, 3, 4], "great": [2, 4], "doesn": [2, 3, 4], "categori": 2, "why": [2, 4], "misinform": 2, "prevent": [2, 4], "factual": 2, "databas": [2, 4], "citat": 2, "tempor": 2, "scientif": 2, "fals": [2, 4], "reduc": [2, 3, 4], "legal": 2, "reput": 2, "protect": 2, "manipul": 2, "unqualifi": 2, "recognit": 2, "medic": 2, "disclaim": 2, "profession": [2, 4], "referr": 2, "mechan": 2, "boundari": 2, "situat": [2, 3], "incorrect": 2, "liabil": 2, "vulner": 2, "standard": 2, "expertis": 2, "util": [2, 3], "bia": [2, 4], "gender": 2, "racial": 2, "cultur": 2, "demograph": 2, "represent": [2, 3], "inclus": [2, 3, 4], "stereotyp": 2, "fair": 2, "reinforc": 2, "societ": 2, "equal": 2, "social": 2, "brand": 2, "privaci": [2, 4], "pii": 2, "anonym": 2, "leakag": 2, "carryov": 2, "regul": [2, 4], "protocol": 2, "confidenti": 2, "breach": 2, "cognit": 2, "multi": [2, 4], "mathemat": 2, "fallaci": 2, "causal": 2, "edg": 2, "think": 2, "mainten": 2, "idiom": 2, "sarcasm": 2, "terminologi": 2, "lingual": 2, "misunderstand": 2, "sophist": [2, 3], "syntax": 2, "scan": 2, "document": [2, 3, 4], "compat": [2, 4], "stabil": 2, "effici": [2, 3, 4], "debt": 2, "scalabl": [2, 3], "failur": 2, "meta": [2, 3], "correct": [2, 4], "feedback": [2, 4], "overconfid": 2, "improv": [2, 3, 4], "clariti": [2, 3, 4], "audienc": 2, "densiti": 2, "transfer": 2, "satisfact": [2, 4], "ethic": 2, "request": [2, 3, 4], "incid": 2, "misus": 2, "moral": 2, "valu": [2, 3, 4], "transpar": [2, 4], "stakehold": 2, "environment": 2, "co2": 2, "emiss": 2, "energi": 2, "consumpt": 2, "per": [2, 3], "server": [2, 4], "locat": 2, "batch": 2, "hardwar": 2, "infer": 2, "sustain": 2, "corpor": 2, "three": 2, "app": 2, "imag": 2, "audio": 2, "etc": [2, 4], "truth": [2, 4], "option": [2, 3, 4], "layer": [2, 3, 4], "repres": [2, 4], "palm": 2, "individu": [2, 3], "target": [2, 4], "further": [2, 3, 4], "see": [2, 4], "avail": [2, 3, 4], "addition": 2, "shown": 2, "fix": [2, 3], "default": [2, 4], "quantifi": 2, "rank": 2, "easi": [2, 3], "addit": [2, 3, 4], "quantit": 2, "among": 2, "aggreg": 2, "plan": [2, 4], "pertain": 2, "previous": [2, 3, 4], "doe": [2, 3, 4], "good": [2, 4], "separ": [2, 3], "synthet": [2, 4], "updat": [2, 3, 4], "reflect": 2, "post": [2, 4], "launch": 2, "timeout": 2, "variat": 2, "maxim": 2, "success": [2, 4], "inter": 2, "rater": 2, "weight": 2, "rel": 2, "priorit": 2, "normal": [2, 4], "absolut": [2, 4], "fail": 2, "confid": [2, 4], "interv": 2, "ti": 2, "veri": 2, "close": 2, "tier": 2, "holist": 2, "built": [2, 4], "mind": 2, "x": 2, "fast": 2, "particularli": [2, 3, 4], "promot": 2, "rapid": 2, "experiment": [2, 4], "iter": [2, 3, 4], "final": [2, 3, 4], "keep": [2, 3], "itself": 2, "confirm": 2, "vi": 2, "later": [2, 4], "chapter": 2, "categor": [2, 4], "broad": [2, 4], "intrins": 2, "extrins": 2, "sequenc": [2, 4], "perplex": 2, "given": [2, 3, 4], "variou": [2, 3, 4], "downstream": [2, 4], "directli": [2, 4], "valuabl": [2, 4], "interest": [2, 3, 4], "sinc": [2, 3, 4], "term": [2, 3], "discrimin": 2, "distinguish": 2, "classifi": 2, "sentiment": [2, 4], "classif": [2, 4], "identifi": [2, 3, 4], "whether": [2, 3, 4], "true": [2, 3, 4], "synthesi": 2, "summar": [2, 3], "log": 2, "discret": 2, "recal": 2, "f1": 2, "match": [2, 4], "exact": 2, "prefix": 2, "translat": 2, "roug": 2, "bleu": 2, "charact": [2, 3, 4], "n": [2, 3], "gram": 2, "being": [2, 4], "guidelin": 2, "subsequ": [2, 4], "section": [2, 3, 4], "short": [2, 3, 4], "wide": [2, 3, 4], "definit": [2, 4], "bilingu": 2, "understudi": 2, "overlap": [2, 3], "favor": [2, 4], "due": [2, 3], "breviti": 2, "penalti": 2, "insensit": 2, "semant": [2, 3], "high": [2, 3], "orient": 2, "gist": 2, "word": [2, 3, 4], "sentenc": [2, 3, 4], "ignor": 2, "equival": 2, "influenc": [2, 4], "meteor": 2, "synonym": 2, "stem": [2, 4], "paraphras": 2, "alongsid": 2, "computation": [2, 3], "expens": 2, "cider": 2, "consensu": 2, "descript": [2, 4], "tf": 2, "idf": 2, "caption": 2, "outsid": 2, "reliant": 2, "corpu": 2, "statist": 2, "ter": 2, "edit": 2, "number": [2, 3, 4], "convert": [2, 4], "hypothesi": 2, "penal": 2, "bertscor": 2, "embed": [2, 3], "bert": 2, "spice": 2, "proposit": 2, "scene": 2, "graph": 2, "emphasi": 2, "onli": [2, 3, 4], "pure": 2, "textual": 2, "As": [2, 3, 4], "analyst": [2, 3], "prepar": [2, 3], "dictionari": [2, 4], "rouge_1": 2, "rouge_2": 2, "ideal": [2, 4], "humana": 2, "expert": [2, 3, 4], "cheaper": 2, "4o": [2, 3, 4], "mini": [2, 3, 4], "evaluate_summari": 2, "unigram": 2, "bigram": 2, "huggingfac": 2, "librari": [2, 3, 4], "absl": 2, "py": 2, "rouge_scor": 2, "generated_summari": 2, "reference_summari": 2, "arg": [2, 3, 4], "dict": [2, 3, 4], "google_bleu": 2, "bleu_scor": 2, "rouge1": 2, "rouge2": 2, "instanc": [2, 3], "arbitrari": 2, "chosen": 2, "sentence1": 2, "cat": 2, "sat": 2, "mat": 2, "sentence2": 2, "ate": 2, "3333333333333333": 2, "7272727272727272": 2, "4444444444444445": 2, "generate_summari": 2, "summir": 2, "correspond": [2, 4], "liner": 2, "excerpt": 2, "evaluate_summary_model": 2, "model_benchmark": 2, "models_test": 2, "benchmark_summari": 2, "model_summari": 2, "evaluation_result": 2, "line": 2, "name": [2, 3, 4], "zip": 2, "annual": 2, "stock": [2, 4], "govern": 2, "forward": 2, "analyz": [2, 3, 4], "statu": 2, "concis": 2, "omit": [2, 4], "essenti": [2, 3, 4], "element": [2, 4], "Its": 2, "adequ": 2, "verbos": 2, "relev": 2, "introduc": [2, 3, 4], "peripher": 2, "quit": [2, 4], "overli": [2, 4], "simplifi": [2, 4], "miss": 2, "convei": [2, 3], "breadth": 2, "Of": 2, "cours": 2, "abov": 2, "vibe": 2, "visualize_prompt_comparison": 2, "visual": 2, "matplotlib": 2, "radar": 2, "plot": 2, "radar_plot": 2, "show": [2, 3, 4], "tmp": 2, "ipykernel_1652501": 2, "940173201": 2, "userwarn": 2, "figurecanvasagg": 2, "thu": 2, "put": 2, "closest": 2, "largest": 2, "deviat": [2, 4], "suggest": [2, 4], "least": 2, "establish": 2, "otherwis": 2, "qualit": 2, "driven": 2, "mention": [2, 4], "might": [2, 3, 4], "fulli": [2, 3], "nuanc": [2, 3, 4], "especi": [2, 3, 4], "those": [2, 3, 4], "primarili": 2, "granular": [2, 3], "altern": [2, 3], "fall": 2, "judg": 2, "themselv": 2, "act": 2, "paper": [2, 4], "main": [2, 3, 4], "instruct": [2, 3, 4], "fine": [2, 4], "tune": [2, 4], "assign": 2, "likert": 2, "style": 2, "pairwis": 2, "ensembl": 2, "repeatedli": 2, "domain": 2, "procedur": 2, "fluenci": 2, "interpret": 2, "refin": 2, "excel": [2, 4], "narr": 2, "flow": [2, 3], "mirror": 2, "express": [2, 4], "similarli": 2, "notabl": [2, 4], "properli": [2, 4], "henc": 2, "worth": 2, "integ": 2, "rubric": 2, "foster": [2, 4], "hollist": 2, "judgeevalu": 2, "enforc": [2, 4], "four": 2, "grammar": [2, 4], "evaluate_with_llm": 2, "candid": 2, "pars": [2, 4], "criterion": 2, "basemodel": [2, 4], "class": [2, 3, 4], "judge_model": 2, "candidate_summari": 2, "specifi": [2, 3, 4], "wa": [2, 4], "written": 2, "grammat": 2, "y": 2, "z": 2, "w": [2, 3], "beta": [2, 4], "response_format": [2, 4], "Then": 2, "benchmark_model": 2, "test_model": 2, "input_text": [2, 3], "tupl": 2, "iphon": [2, 4], "mac": [2, 4], "ipad": [2, 4], "incorpor": 2, "regard": 2, "obtain": [2, 4], "respect": 2, "regist": 2, "approxim": [2, 4], "6": [2, 3, 4], "trillion": [2, 4], "held": [2, 4], "affili": [2, 4], "billion": 2, "outstand": [2, 4], "octob": [2, 4], "18": [2, 4], "7": [2, 3], "8": [2, 3], "evals_list": 2, "1775618912": 2, "14": [2, 4], "some": [2, 3, 4], "achiev": [2, 4], "variant": 2, "slightli": 2, "indic": [2, 4], "drift": 2, "had": 2, "lowest": 2, "overal": [2, 3, 4], "drop": 2, "substanti": 2, "gradient": 2, "visibl": 2, "mark": 2, "degrad": [2, 4], "firstli": 2, "overhead": 2, "neglect": 2, "exhibit": 2, "prefer": [2, 4], "posit": [2, 3, 4], "egocentr": 2, "tight": 2, "small": [2, 4], "field": [2, 4], "financ": 2, "law": 2, "suitabl": 2, "serv": [2, 3, 4], "aproach": 2, "significantli": [2, 3], "workflow": [2, 4], "assessor": 2, "intens": [2, 4], "aplic": 2, "aim": [2, 3, 4], "clearli": [2, 4], "earlier": 2, "depict": [2, 4], "higher": 2, "correl": 2, "were": [2, 4], "multilingu": 2, "golden": 2, "recruit": 2, "languang": 2, "extens": [2, 3, 4], "arena": 2, "vote": 2, "made": [2, 3, 4], "under": [2, 4], "blind": 2, "randomli": 2, "pair": 2, "submit": 2, "actual": [2, 3, 4], "loop": 2, "customiz": 2, "irrelev": 2, "unhelp": 2, "sometim": [2, 4], "though": [2, 4], "occasion": 2, "regularli": 2, "inquiri": 2, "rare": 2, "inaccuraci": 2, "highli": [2, 4], "perfectli": 2, "cater": 2, "polici": 2, "benefit": [2, 4], "critiqu": 2, "elo": 2, "democrat": [2, 4], "simul": [2, 4], "thought": [2, 4], "exam": 2, "probe": 2, "certifi": 2, "identif": 2, "histori": 2, "prioriti": 2, "intellig": 2, "move": [2, 3], "began": 2, "2018": 2, "introduct": [2, 3], "glue": 2, "entail": 2, "baselin": 2, "superglu": 2, "2019": 2, "expand": 2, "deeper": [2, 3], "successor": 2, "grew": 2, "broader": 2, "big": 2, "bench": 2, "2021": 2, "turn": 2, "200": 2, "span": 2, "arithmet": 2, "collabor": 2, "truthfulqa": 2, "accur": [2, 4], "decept": 2, "increasingli": [2, 4], "stanford": 2, "helm": 2, "multidimension": 2, "concern": 2, "surround": [2, 4], "emphas": [2, 4], "multitask": 2, "multidisciplinari": 2, "57": 2, "humanev": 2, "lmsy": 2, "2023": 2, "brought": 2, "dialogu": 2, "len": [2, 3], "dai": [2, 4], "groundbreak": 2, "arc": 2, "prize": 2, "steadi": 2, "progress": [2, 3], "meaning": [2, 3, 4], "bar": 2, "ambit": 2, "so": [2, 4], "track": 2, "replic": [2, 4], "find": [2, 3, 4], "industri": [2, 4], "chatbot": 2, "formerli": 2, "direct": 2, "live": 2, "gather": 2, "000": [2, 4], "assist": [2, 4], "alpacaev": 2, "mt": 2, "Their": [2, 4], "came": 2, "center": [2, 4], "mike": 2, "knoop": 2, "co": 2, "founder": 2, "zapier": 2, "fran\u00e7oi": 2, "chollet": 2, "creator": 2, "agi": 2, "kera": 2, "narrow": 2, "suffici": [2, 4], "capac": 2, "genuin": 2, "accord": 2, "wrong": 2, "econom": 2, "acquir": 2, "skill": 2, "five": 2, "old": 2, "possess": 2, "count": [2, 3], "elementari": 2, "physic": 2, "novelti": 2, "puzzl": 2, "novel": 2, "someth": 2, "wouldn": 2, "vast": 2, "interpol": 2, "memori": [2, 3], "synthes": 2, "fly": 2, "brute": 2, "possibl": [2, 4], "million": 2, "seri": 2, "minim": [2, 4], "prior": 2, "submiss": 2, "pixel": 2, "perfect": 2, "color": 2, "modern": [2, 3, 4], "meaningfulli": 2, "fourrier": 2, "lightweight": [2, 4], "varieti": 2, "bespok": 2, "via": [2, 4], "sdk": 2, "cli": 2, "been": 2, "extract": [2, 3, 4], "autoregress": 2, "conduct": 2, "sub": 2, "liter": 2, "disturb": 2, "zero": 2, "varianc": 2, "yt": 2, "ut": 2, "uncondit": 2, "33": 2, "suppos": 2, "p": 2, "08": [2, 4], "exactli": [2, 4], "ii": [2, 4], "iv": 2, "iii": 2, "c": [2, 4], "consequ": 2, "ol": 2, "heteroscedast": 2, "regress": 2, "ineffici": 2, "wish": 2, "lag": 2, "var": 2, "bivari": 2, "acceler": 2, "evaluation_track": 2, "evaluationtrack": 2, "model_config": 2, "basemodelconfig": 2, "parallelismmanag": 2, "pipelineparamet": 2, "envconfig": 2, "is_accelerate_avail": 2, "datetim": 2, "timedelta": 2, "initprocessgroupkwarg": 2, "create_evaluation_pipelin": 2, "output_dir": 2, "cache_dir": 2, "pretrain": 2, "dtype": 2, "float16": 2, "max_sampl": 2, "kwargs_handl": 2, "second": [2, 3], "3000": 2, "els": [2, 3], "none": 2, "save_detail": 2, "push_to_hub": 2, "pipeline_param": 2, "launcher_typ": 2, "env_config": 2, "override_batch_s": 2, "use_chat_templ": 2, "trust_remote_cod": 2, "pipeline_paramet": 2, "schemat": [2, 3], "vllm": [2, 4], "tgi": 2, "instanti": 2, "storag": 2, "local": [2, 3, 4], "push": 2, "hub": 2, "parallel": 2, "temporari": 2, "maximum": [2, 3], "num_few_shot": 2, "automat": 2, "string": [2, 4], "vertic": 2, "few": [2, 3, 4], "binari": 2, "flag": 2, "bigbench": 2, "winogrand": 2, "hellaswag": 2, "nlp": 2, "choos": 2, "1b": 2, "save": [2, 3], "save_and_push_result": 2, "show_result": 2, "model_arg": 2, "download": 2, "remot": 2, "send": [2, 4], "serverless": 2, "dedic": [2, 4], "id": 2, "inference_server_address": 2, "inference_server_auth": 2, "model_id": 2, "null": 2, "bash": 2, "command": 2, "model_config_path": 2, "path": [2, 3], "endpoint_model": 2, "yaml": [2, 4], "llama3": [2, 3], "qwen2": [2, 4], "smollm2": 2, "describ": 2, "3b": 2, "alibaba": [2, 4], "5b": [2, 4], "hui": 2, "yang": 2, "compact": 2, "360m": 2, "allal": 2, "9": 2, "trend": [2, 4], "cluster": 2, "own": [2, 3], "degre": 2, "noteworthi": 2, "superior": 2, "taken": [2, 4], "grain": [2, 4], "salt": [2, 4], "100": [2, 4], "give": 2, "trade": [2, 4], "off": [2, 3, 4], "flexibl": [2, 3, 4], "exponenti": 2, "growth": 2, "hug": [2, 4], "ecosystem": 2, "modular": 2, "visit": 2, "offici": 2, "alb": 2, "24": [2, 4], "loubna": 2, "ben": 2, "anton": 2, "lozhkov": 2, "eli": 2, "bakouch": 2, "gabriel": 2, "mart\u00edn": 2, "bl\u00e1zquez": 2, "lewi": 2, "tunstal": 2, "agust\u00edn": 2, "piquer": 2, "andr": 2, "marafioti": 2, "cyril": 2, "zakka": 2, "leandro": 2, "von": 2, "werra": 2, "thoma": 2, "wolf": 2, "are24": 2, "judgearena": 2, "fac24a": 2, "wiki": [2, 4], "fac24b": 2, "fac24c": 2, "doc": [2, 3, 4], "model_doc": 2, "gpt2": 2, "fac24d": 2, "cookbook": 2, "en": [2, 4], "llm_judg": 2, "fac24": 2, "fhwt23": 2, "cl\u00e9mentin": 2, "nathan": 2, "habib": 2, "url": [2, 4], "hyc": 2, "binyuan": 2, "jian": 2, "zeyu": 2, "cui": 2, "jiaxi": 2, "dayiheng": 2, "liu": [2, 4], "lei": 2, "zhang": 2, "tianyu": 2, "jiajun": 2, "bowen": 2, "yu": 2, "kai": 2, "dang": 2, "coder": 2, "arxiv": [2, 4], "preprint": [2, 4], "2409": 2, "12186": 2, "lx": 2, "zhen": 2, "xiaohan": 2, "xu": 2, "tao": 2, "shen": 2, "jia": 2, "chen": [2, 4], "gu": 2, "yuxuan": 2, "lai": 2, "chongyang": 2, "shuai": 2, "ma": 2, "nlg": 2, "org": [2, 4], "ab": [2, 4], "2401": 2, "07103": 2, "wtb": 2, "22": 2, "jason": 2, "yi": [2, 4], "tai": 2, "rishi": 2, "bommasani": 2, "colin": 2, "raffel": 2, "barret": 2, "zoph": 2, "sebastian": 2, "borgeaud": 2, "dani": 2, "yogatama": 2, "maarten": 2, "bosma": 2, "denni": 2, "zhou": 2, "donald": 2, "metzler": 2, "ed": 2, "h": 2, "chi": [2, 4], "tatsunori": 2, "hashimoto": 2, "oriol": 2, "vinyal": 2, "perci": 2, "liang": 2, "jeff": 2, "dean": 2, "william": 2, "fedu": 2, "2206": 2, "07682": 2, "yyh": 2, "baosong": 2, "bo": 2, "zheng": 2, "chengpeng": 2, "chengyuan": 2, "fei": 2, "huang": 2, "guant": 2, "dong": 2, "haoran": 2, "huan": 2, "lin": [2, 4], "jialong": 2, "tang": 2, "jialin": 2, "wang": 2, "jianhong": 2, "tu": 2, "jianwei": 2, "jianxin": 2, "jin": 2, "jingren": 2, "jinz": 2, "bai": 2, "jinzheng": 2, "junyang": 2, "keme": 2, "lu": 2, "keqin": 2, "kexin": 2, "mei": 2, "mingfeng": 2, "xue": 2, "na": 2, "ni": 2, "pei": 2, "peng": 2, "ru": 2, "rui": [2, 4], "men": 2, "ruiz": 2, "gao": 2, "runji": 2, "shiji": 2, "sinan": 2, "tan": 2, "tianhang": 2, "zhu": 2, "tianhao": 2, "wenbin": 2, "ge": 2, "xiaodong": 2, "deng": 2, "xiaohuan": 2, "xingzhang": 2, "ren": 2, "xinyu": 2, "xipin": 2, "xuancheng": 2, "fan": 2, "yao": 2, "yichang": 2, "wan": 2, "yunfei": 2, "chu": 2, "yuqiong": 2, "zhenru": 2, "zhihao": 2, "2407": 2, "10671": 2, "huggingface24": 2, "space": 2, "12": [2, 3], "06": [2, 4], "metaai24": 2, "promptfoo24": 2, "toolkit": 2, "www": 2, "dev": 2, "go": [3, 4], "far": 3, "possibli": 3, "eliot": 3, "english": 3, "rule": [3, 4], "thumb": 3, "\u00be": 3, "max_output_token": 3, "4096": 3, "16384": 3, "contrari": 3, "surpass": 3, "stop": 3, "mid": 3, "truncat": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "11b": 3, "v1": 3, "128000": 3, "5e": 3, "sonnet": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "0613": 3, "6e": 3, "04": 3, "09": 3, "1e": 3, "gemini": 3, "flash": 3, "002": 3, "1048576": 3, "pro": 3, "2097152": 3, "05e": 3, "pose": [3, 4], "incomplet": 3, "articl": 3, "abruptli": 3, "cut": 3, "disrupt": 3, "shallow": 3, "thorough": 3, "receiv": 3, "partial": 3, "dissatisfact": 3, "frustrat": 3, "educ": 3, "creation": 3, "feasibl": 3, "split": 3, "previou": [3, 4], "10k": 3, "diagram": 3, "charactertextsplitt": 3, "tiktoken": 3, "sequenti": 3, "chain": 3, "newlin": 3, "broadli": [3, 4], "decid": 3, "want": 3, "sure": [3, 4], "lost": 3, "cheap": 3, "speciali": 3, "advantag": [3, 4], "naiv": 3, "period": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "manner": [3, 4], "talk": 3, "theme": 3, "topic": [3, 4], "splitter": 3, "markdown": 3, "html": [3, 4], "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "persona": 3, "assum": 3, "task": [3, 4], "action": 3, "langchain_cor": [3, 4], "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": [3, 4], "from_templ": 3, "llmchain": 3, "togeth": 3, "parser": [3, 4], "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": [3, 4], "llm_chain": [3, 4], "api_key_label": 3, "upper": 3, "_api_kei": 3, "api_kei": 3, "get_dynamic_prompt_templ": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "origin": [3, 4], "part": [3, 4], "total": [3, 4], "param": 3, "dynamic_prompt_param": 3, "copi": 3, "elif": 3, "last": [3, 4], "merg": 3, "concaten": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "dinam": 3, "priovid": 3, "enumer": 3, "invok": [3, 4], "cummul": 3, "join": 3, "max_chunk_s": 3, "max_chunk_overlap": 3, "latest": [3, 4], "readabl": 3, "apple_report": 3, "300": 3, "disclos": [3, 4], "state": [3, 4], "luation": 3, "oblig": 3, "cash": 3, "disciplin": 3, "smooth": 3, "upon": 3, "subhead": 3, "adher": [3, 4], "revenu": [3, 4], "segment": [3, 4], "liquid": 3, "capit": [3, 4], "despit": [3, 4], "depth": 3, "overlook": 3, "mitig": [3, 4], "fit": [3, 4], "within": [3, 4], "preserv": 3, "easier": [3, 4], "preprocess": 3, "enhanc": [3, 4], "necessit": 3, "meticul": 3, "retain": 3, "necessari": 3, "seamlessli": 3, "circumv": 3, "therebi": 3, "escal": 3, "frequenc": 3, "volum": 3, "bottleneck": 3, "latenc": 3, "friendli": 3, "mustafa": 3, "suleyman": 3, "infinit": 3, "amount": [3, 4], "fewer": 3, "compress": 3, "condens": 3, "adjust": [3, 4], "constrain": [3, 4], "collect": 3, "versatil": 3, "drive": [3, 4], "grace": 3, "fallback": 3, "empow": 3, "crucial": [3, 4], "stai": 3, "full": [3, 4], "langchain24": 3, "how_to": 3, "07": [3, 4], "freedom": 4, "thrive": 4, "julia": 4, "cameron": 4, "easili": 4, "notebook": 4, "overrid": 4, "response_cont": 4, "wow": 4, "lot": 4, "breakdown": 4, "stream": 4, "portfolio": 4, "impress": 4, "notic": 4, "march": 4, "29": 4, "huge": 4, "investor": 4, "figur": 4, "compli": 4, "ye": 4, "date": 4, "serious": 4, "is_json": 4, "myjson": 4, "except": 4, "valueerror": 4, "lack": 4, "googl": 4, "survei": 4, "51": 4, "trial": 4, "elicit": 4, "consum": 4, "wrangl": 4, "conform": 4, "ad": 4, "hoc": 4, "streamlin": 4, "modul": 4, "dataset": 4, "unwant": 4, "neg": 4, "ui": 4, "restrict": 4, "mobil": 4, "devic": 4, "overflow": 4, "overwhelm": 4, "twitter": 4, "youtub": 4, "impos": 4, "publish": 4, "successfulli": 4, "adopt": 4, "emploi": 4, "schema": 4, "blueprint": 4, "nativ": 4, "regular": 4, "json_format": 4, "person1": 4, "alic": 4, "q1": 4, "20": 4, "person2": 4, "bob": 4, "net": 4, "margin": 4, "materi": 4, "nest": 4, "todai": 4, "programmat": 4, "thellm": 4, "unend": 4, "whitespac": 4, "until": 4, "forget": 4, "throw": 4, "appear": 4, "somewher": 4, "json_object": 4, "628": 4, "553": 4, "sheer": 4, "115": 4, "823": 4, "circul": 4, "plai": 4, "vertex": 4, "releas": 4, "suppli": 4, "worri": 4, "enum": 4, "No": 4, "incorrectli": 4, "refus": 4, "simpler": 4, "strongli": 4, "entiti": 4, "place": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "hint": 4, "attribut": 4, "prompt_extract": 4, "sec_extract": 4, "nasdaq": 4, "llc": 4, "washington": 4, "cupertino": 4, "usabl": 4, "beg": 4, "with_structured_output": 4, "runnabl": 4, "typeddict": 4, "qu": 4, "langchain_openai": 4, "chatopenai": 4, "chatprompttempl": 4, "extract_from_sec_filing_langchain": 4, "structured_llm": 4, "from_messag": 4, "sec_extraction_langchain": 4, "found": 4, "hood": 4, "logit": 4, "raw": 4, "neural": 4, "network": 4, "regex": 4, "strong": 4, "enough": 4, "qwen": 4, "top": 4, "label": 4, "unexpect": 4, "malform": 4, "pass": 4, "sec_extraction_outlin": 4, "zsp": 4, "zicorp": 4, "phenomenon": 4, "popular": 4, "cpp": 4, "gbnf": 4, "ggml": 4, "bnf": 4, "ggerganov": 4, "accomplish": 4, "formal": 4, "backu": 4, "naur": 4, "wikipedia": 4, "contributor": 4, "strictli": 4, "soon": 4, "curl": 4, "fssl": 4, "sh": 4, "did": 4, "extract_entities_from_sec_fil": 4, "suffix": 4, "ollama_structured_output_prompt_suffix": 4, "ollama_structured_output_temperatur": 4, "mistral": 4, "llama2": 4, "uncensor": 4, "model_json_schema": 4, "response_json": 4, "AND": 4, "wrapper": 4, "exllama2": 4, "mlx": 4, "lm": 4, "enterpris": 4, "commerci": 4, "medium": 4, "low": 4, "done": 4, "know": 4, "chanc": 4, "connect": 4, "correctli": 4, "area": 4, "mix": 4, "famili": 4, "furthermor": 4, "nonetheless": 4, "evid": 4, "studi": 4, "wrap": 4, "map": 4, "gemma": 4, "uncov": 4, "wors": 4, "extran": 4, "dispar": 4, "preval": 4, "outdat": 4, "rapidli": 4, "fashion": 4, "remark": 4, "me": 4, "speak": 4, "freeli": 4, "tam": 4, "aider": 4, "decod": 4, "hinder": 4, "outweigh": 4, "team": 4, "rebutt": 4, "argu": 4, "v": 4, "compel": 4, "reproduct": 4, "paint": 4, "pictur": 4, "publicli": 4, "independ": 4, "verif": 4, "dottxt": 4, "flaw": 4, "believ": 4, "led": 4, "inaccur": 4, "reconcil": 4, "uneven": 4, "didn": 4, "conflat": 4, "argument": 4, "drawback": 4, "unlock": 4, "wider": 4, "thank": 4, "pfiffer": 4, "hi": 4, "aid24": 4, "dot24": 4, "sai": 4, "demo": 4, "tree": 4, "gge24": 4, "blob": 4, "readm": 4, "md": 4, "llf": 4, "michael": 4, "xieyang": 4, "frederick": 4, "alexand": 4, "j": 4, "fiannaca": 4, "terri": 4, "koo": 4, "luca": 4, "dixon": 4, "carri": 4, "cai": 4, "ea": 4, "york": 4, "ny": 4, "usa": 4, "machineri": 4, "doi": 4, "1145": 4, "3613905": 4, "3650756": 4, "ln": 4, "xuan": 4, "hai": 4, "nguyen": 4, "ngoc": 4, "tiviati": 4, "sim": 4, "hieu": 4, "dao": 4, "shafiq": 4, "joti": 4, "kenji": 4, "kawaguchi": 4, "nanci": 4, "min": 4, "yen": 4, "kan": 4, "2408": 4, "08656": 4, "out24": 4, "io": 4, "twt": 4, "zhi": 4, "cheng": 4, "kuang": 4, "wu": 4, "tsai": 4, "chieh": 4, "hung": 4, "lee": 4, "yun": 4, "nung": 4, "02442": 4, "wikipediacontributors24": 4, "wiktionari": 4, "naur_form": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1, 4], "content": [0, 2, 3, 4], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 1], "practic": [0, 1, 4], "approach": 0, "note": 0, "perspect": 0, "who": 0, "thi": 0, "book": 0, "i": 0, "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "python": 0, "setup": 0, "api": [0, 4], "kei": [0, 2, 3], "configur": 0, "code": 0, "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "about": 0, "author": 0, "": 0, "tame": 1, "llm": [1, 2], "guid": 1, "pitfal": 1, "open": 1, "sourc": 1, "softwar": [1, 2], "chapter": 1, "1": [1, 3], "2": [1, 3], "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "3": [1, 3], "input": 1, "size": [1, 3], "length": [1, 3], "limit": [1, 3], "4": [1, 3], "5": 1, "The": [1, 2], "eval": [1, 2], "gap": [1, 2], "6": 1, "hallucin": 1, "realiti": 1, "7": 1, "safeti": 1, "concern": 1, "8": 1, "cost": [1, 3], "factor": 1, "9": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "tool": [1, 2, 4], "resourc": 1, "non": 2, "determinist": 2, "machin": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "emerg": 2, "properti": 2, "problem": [2, 3, 4], "statement": [2, 3, 4], "tradit": 2, "v": 2, "design": 2, "applic": 2, "test": 2, "requir": 2, "matrix": 2, "conceptu": 2, "overview": 2, "consider": [2, 3], "metric": 2, "evalu": 2, "gener": [2, 3], "task": 2, "model": [2, 3], "base": [2, 3], "human": 2, "benchmark": 2, "leaderboard": 2, "lightev": 2, "mmlu": 2, "econometr": 2, "dataset": 2, "famili": 2, "us": 2, "langchain": [2, 4], "promptfoo": 2, "refer": [2, 3, 4], "what": 3, "ar": 3, "token": 3, "comparison": [3, 4], "across": 3, "chunk": 3, "contextu": 3, "link": 3, "long": 3, "form": 3, "step": 3, "write": 3, "prompt": [3, 4], "templat": 3, "construct": 3, "dynam": 3, "paramet": 3, "report": 3, "exampl": 3, "usag": 3, "discuss": [3, 4], "implic": 3, "futur": 3, "conclus": [3, 4], "user": 4, "need": 4, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "specif": 4, "json": 4, "mode": 4, "outlin": 4, "ollama": 4, "compar": 4, "framework": 4, "best": 4, "research": 4, "ongo": 4, "debat": 4, "acknowledg": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"], [4, "introduction"]], "Contents": [[0, "contents"], [2, "contents"], [3, "contents"], [4, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author(s)": [[0, "about-the-author-s"]], "Taming LLMs": [[1, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[1, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Wrestling with Structured Output": [[1, "chapter-2-wrestling-with-structured-output"]], "Chapter 3: Input Size and Length Limitations": [[1, "chapter-3-input-size-and-length-limitations"]], "Chapter 4: Output Size and Length Limitations": [[1, "chapter-4-output-size-and-length-limitations"]], "Chapter 5: The Evals Gap": [[1, "chapter-5-the-evals-gap"]], "Chapter 6: Hallucination: The Reality Gap": [[1, "chapter-6-hallucination-the-reality-gap"]], "Chapter 7: Safety Concerns": [[1, "chapter-7-safety-concerns"]], "Chapter 8: The Cost Factor": [[1, "chapter-8-the-cost-factor"]], "Chapter 9: Breaking Free from Cloud Providers": [[1, "chapter-9-breaking-free-from-cloud-providers"]], "Appendix A: Tools and Resources": [[1, "appendix-a-tools-and-resources"]], "The Evals Gap": [[2, "the-evals-gap"]], "Non-Deterministic Machines": [[2, "non-deterministic-machines"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Emerging Properties": [[2, "emerging-properties"]], "Problem Statement": [[2, "problem-statement"], [3, "problem-statement"], [4, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[2, "evals-table"]], "Evals Design": [[2, "evals-design"]], "LLM Application Testing Requirements Matrix": [[2, "validation-requirements"]], "Conceptual Overview": [[2, "conceptual-overview"]], "Design Considerations": [[2, "design-considerations"]], "Metrics": [[2, "metrics"]], "Key Metrics for Evaluating Generative Tasks": [[2, "key-metrics"]], "Evaluators": [[2, "evaluators"]], "Model-Based Evaluation": [[2, "model-based-evaluation"]], "Human-Based Evaluation": [[2, "human-based-evaluation"]], "Evaluating Evaluators": [[2, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[2, "benchmarks-and-leaderboards"]], "Tools": [[2, "tools"]], "LightEval": [[2, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[2, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[2, "model-families"]], "LangChain": [[2, "langchain"], [4, "langchain"]], "PromptFoo": [[2, "promptfoo"]], "References": [[2, "references"], [3, "references"], [4, "references"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Token Cost and Length Limitation Comparison Across Key Models": [[3, "token-cost-table"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Step 2: Writing the Base Prompt Template": [[3, "step-2-writing-the-base-prompt-template"]], "Step 3: Constructing Dynamic Prompt Parameters": [[3, "step-3-constructing-dynamic-prompt-parameters"]], "Step 4: Generating the Report": [[3, "step-4-generating-the-report"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"], [4, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "User Needs": [[4, "user-needs"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "JSON Mode": [[4, "json-mode"]], "Outlines": [[4, "outlines"]], "Ollama": [[4, "ollama"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[4, "structured-output-frameworks"]], "Best Practices": [[4, "best-practices"]], "Research and Ongoing Debate": [[4, "research-and-ongoing-debate"]], "Acknowledgements": [[4, "acknowledgements"]]}, "indexentries": {}}) \ No newline at end of file diff --git a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb index d37a011..ff080c1 100644 --- a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb +++ b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "8e40cf5d", + "id": "cc6b3dc4", "metadata": {}, "source": [ "(intro)=\n", diff --git a/tamingllms/_build/jupyter_execute/notebooks/output_size_limit.ipynb b/tamingllms/_build/jupyter_execute/notebooks/output_size_limit.ipynb index 64cf950..fdbd3b3 100644 --- a/tamingllms/_build/jupyter_execute/notebooks/output_size_limit.ipynb +++ b/tamingllms/_build/jupyter_execute/notebooks/output_size_limit.ipynb @@ -90,7 +90,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Here, we will utilize `langchain` for a content-aware sentence-splitting strategy for chunking. We will use the `CharacterTextSplitter` with `tiktoken` as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model." + "Here, we will utilize `langchain` for a content-aware sentence-splitting strategy for chunking. Langchain offers several text splitters {cite}`langchain_text_splitters` such as JSON-, Markdown- and HTML-based or split by token. We will use the `CharacterTextSplitter` with `tiktoken` as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model." ] }, { @@ -471,8 +471,9 @@ "\n", "\n", "## References\n", - "\n", - "- [LangChain Text Splitter](https://langchain.readthedocs.io/en/latest/modules/text_splitter.html)." + "```{bibliography}\n", + ":filter: docname in docnames\n", + "```" ] }, { diff --git a/tamingllms/notebooks/output_size_limit.ipynb b/tamingllms/notebooks/output_size_limit.ipynb index 6a79bba..0bc840a 100644 --- a/tamingllms/notebooks/output_size_limit.ipynb +++ b/tamingllms/notebooks/output_size_limit.ipynb @@ -90,7 +90,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Here, we will utilize `langchain` for a content-aware sentence-splitting strategy for chunking. We will use the `CharacterTextSplitter` with `tiktoken` as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model." + "Here, we will utilize `langchain` for a content-aware sentence-splitting strategy for chunking. Langchain offers several text splitters {cite}`langchain_text_splitters` such as JSON-, Markdown- and HTML-based or split by token. We will use the `CharacterTextSplitter` with `tiktoken` as our tokenizer to count the number of tokens per chunk which we can use to ensure that we do not surpass the input token limit of our model." ] }, { @@ -471,8 +471,9 @@ "\n", "\n", "## References\n", - "\n", - "- [LangChain Text Splitter](https://langchain.readthedocs.io/en/latest/modules/text_splitter.html)." + "```{bibliography}\n", + ":filter: docname in docnames\n", + "```" ] }, { diff --git a/tamingllms/references.bib b/tamingllms/references.bib index 5dcbdca..db98c91 100644 --- a/tamingllms/references.bib +++ b/tamingllms/references.bib @@ -227,4 +227,12 @@ @article{long2024llms author={Long, Do Xuan and Ngoc, Hai Nguyen and Sim, Tiviatis and Dao, Hieu and Joty, Shafiq and Kawaguchi, Kenji and Chen, Nancy F and Kan, Min-Yen}, journal={arXiv preprint arXiv:2408.08656}, year={2024} -} \ No newline at end of file +} + +@misc{langchain_text_splitters, + title={Text Splitters - LangChain Documentation}, + author={{LangChain}}, + year={2024}, + howpublished={\url{https://python.langchain.com/docs/how_to/#text-splitters}}, + note={Accessed: 12/07/2024} +}
              Table 3.1 Structured Output Frameworks Comparison