From 36bf6451c45fde104ee88c2491aa66a766e9250a Mon Sep 17 00:00:00 2001 From: Josh Reini Date: Fri, 21 Jun 2024 16:48:34 -0400 Subject: [PATCH] Context filtering guardrails (#1192) * fix select_context to call return individual items, not collected list * display utilities, context filtering, oh my * Update trulens_eval/trulens_eval/tru_chain.py Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * undo bad ellipsis change * change moved import * update llama quickstart * get llama working * llama updates * add docstring examples * guardrails docs page * allow tabbed content setting in mkdocs yml * formatting * api ref links * api ref files for guardrails * api ref to guardrails in mkdocs * update in trubot * fix typo * add missing init * api ref updates * docstring updates * docstring updates * base context filter for custom apps * guardrails in quickstart * make display util more efficient * switch openai model to std * drop unneccessary import * docs for base guardrails * remove unneccessary name * drop unneccessary args * using sentence * fix select_context * langchain quickstart updates * fix link * langchain feedback type error handling * llama feedback type error handling * base feedback type error handling * pass context as default argument to lambda function in base Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * revert ellipsis change * fix test failure * move docstrings, simplify * accept directionality from feedback * set wraper.__dict__ too * filter on demand * format, remove unused imports * format * format * assert llama-index installed * assert langchain installed * test llam aguardrails optional mod * format * copy signature instead of dict * add note --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- docs/trulens_eval/api/guardrails/index.md | 3 + docs/trulens_eval/api/guardrails/langchain.md | 3 + docs/trulens_eval/api/guardrails/llama.md | 3 + .../guardrail_context_filtering.png | Bin 0 -> 103785 bytes docs/trulens_eval/guardrails/index.md | 93 ++ .../guardrails/simple_guardrail_flow.png | Bin 0 -> 31962 bytes mkdocs.yml | 10 + .../end2end_apps/trubot/trubot_example.ipynb | 2 +- .../quickstart/langchain_quickstart.ipynb | 884 ++++++++++++++++-- .../quickstart/llama_index_quickstart.ipynb | 710 +++++++++++++- .../examples/quickstart/quickstart.ipynb | 557 ++++++++++- trulens_eval/tests/unit/static/test_static.py | 5 +- .../trulens_eval/guardrails/__init__.py | 0 trulens_eval/trulens_eval/guardrails/base.py | 60 ++ .../trulens_eval/guardrails/langchain.py | 122 +++ trulens_eval/trulens_eval/guardrails/llama.py | 103 ++ trulens_eval/trulens_eval/tru_chain.py | 7 +- trulens_eval/trulens_eval/tru_llama.py | 2 +- trulens_eval/trulens_eval/utils/display.py | 20 + trulens_eval/trulens_eval/utils/langchain.py | 71 +- trulens_eval/trulens_eval/utils/llama.py | 72 +- 21 files changed, 2436 insertions(+), 291 deletions(-) create mode 100644 docs/trulens_eval/api/guardrails/index.md create mode 100644 docs/trulens_eval/api/guardrails/langchain.md create mode 100644 docs/trulens_eval/api/guardrails/llama.md create mode 100644 docs/trulens_eval/guardrails/guardrail_context_filtering.png create mode 100644 docs/trulens_eval/guardrails/index.md create mode 100644 docs/trulens_eval/guardrails/simple_guardrail_flow.png create mode 100644 trulens_eval/trulens_eval/guardrails/__init__.py create mode 100644 trulens_eval/trulens_eval/guardrails/base.py create mode 100644 trulens_eval/trulens_eval/guardrails/langchain.py create mode 100644 trulens_eval/trulens_eval/guardrails/llama.py create mode 100644 trulens_eval/trulens_eval/utils/display.py diff --git a/docs/trulens_eval/api/guardrails/index.md b/docs/trulens_eval/api/guardrails/index.md new file mode 100644 index 000000000..b4fb1930b --- /dev/null +++ b/docs/trulens_eval/api/guardrails/index.md @@ -0,0 +1,3 @@ +# TruLens Guardrails + +::: trulens_eval.guardrails.base \ No newline at end of file diff --git a/docs/trulens_eval/api/guardrails/langchain.md b/docs/trulens_eval/api/guardrails/langchain.md new file mode 100644 index 000000000..c2d1c1337 --- /dev/null +++ b/docs/trulens_eval/api/guardrails/langchain.md @@ -0,0 +1,3 @@ +# Guardrails with Langchain + +::: trulens_eval.guardrails.langchain \ No newline at end of file diff --git a/docs/trulens_eval/api/guardrails/llama.md b/docs/trulens_eval/api/guardrails/llama.md new file mode 100644 index 000000000..8ea9b20d1 --- /dev/null +++ b/docs/trulens_eval/api/guardrails/llama.md @@ -0,0 +1,3 @@ +# Guardrails with Llama-Index + +::: trulens_eval.guardrails.llama \ No newline at end of file diff --git a/docs/trulens_eval/guardrails/guardrail_context_filtering.png b/docs/trulens_eval/guardrails/guardrail_context_filtering.png new file mode 100644 index 0000000000000000000000000000000000000000..e753ec682194f2acec4201e4f756ed50576b782f GIT binary patch literal 103785 zcmeFZWmJ@F7cdM6NC=W*5F;JZ-Gg*WcL+##OEaQU($Xp2Nap|wN=e7iAl=>h-FTk! z9E0=x`qum7T`y|^_so6e?!B+QuK;;DaSXJFXmD_F7?Khqig0kqLU3>hE-1*rH)NNG zDR6M`dS=4H@{+>BAbEQmV>3%5I5_HXt1y1)=I3~Co>^LbQgy_#vu&Gs{y{1L>2~&* z5Tai%t(Q!end}=jPg%u4uH|_~EHx@YWbuk1-z%_C&dj{(ZO#uMP2H`89IGMpm_6I- zCHoriGiAeXcMzyGijv|IqSRDj^V_UX{ZmN{QA}%_74vGrWXHps=EILsp($8(&d!EQ zt-*!%`PlOWjAwJ+OLcjSdbW=Jcg;mjD$ylS4JuYyKDoCDa}bGofJU}YWNf>aENqOK zBPkomhzEl-laB+qlGnC_F&X0UFV>LlJjEp4TY41B_)CXN{04EM%E6dHjXT@X{W$YFf(O!M|D{l9zz=|2K^T{21X1ot*)nr zgX4S213X$8IqHL6T3K2<@Vw+F{n3I4c)tFckrec!iK7KSsk*E@NZ7{S2*km_#K1%< zfCd7A`0QU8^C*gl{ii$diJ#Qe(b1NNkjRWvnSqJ%zheVk`L4g^kvDs3WT_!yW(9x;m_vYrm6`8H`#;|NGviM^)&J?q z%)#_a*PmYf-c`lH$X?jS3YgPT;2+fdXWpM*{-+}!xi&{_!mUXaO`n#{VKs z0L|r0Jsl1X3@0ffr2G@hS9Hgd;vr&z;o$$}59ll6tik$AF2a9#|D$6Q4hZG*ZTNp1188HQ z=mV0FNmj)~`yY)^UTr`4_m#jwV(~yIIFZ>=zEA&kxp45WKpx2dYd@4oD6d-P%@$Js zMN-lmD^K{|Aykf%E^U!w+ZHH#KI`!E!c5BLenz7?#Y>_y+ohbfEBEtcYlo+Cj14a^I1Tlls6R z3{lxmFPGR~#oDR#Lo`ifb0hdmaBt`gk4XDCRVdViQkMK!0kt`Af-TcJqTIZ{oYtki z0SgTJjHGHksyro3gl|8+Y=^Vi_ZU&Aqumt(hX5?@*FRY82s&GJWD#}Z)0PN2T1)LD zo2MVFXl$^-safa+ERZnNf$RV}Q0Zlf{&(5e6yswdRY36wc9y5OvVbT= zD=Ts9rqRFSp)7qVu{}(v{4rRb9w2oOQf|5ZcjDiH=mQSM?ImB)Y(rI?ljFbr?pF~1 z(k%-AIk|)p(kSsCH2RMuB~=7HR&-lt+L_miVwV!ETy2L$nN&5&+d z1Oy93*&S>C(EF02K=Rqs=OVb)2xM*tQ9*7;4qBh35wUXG5$LKcOEJAa@0+PIY26xQ zyqG6=hnCw;f)+yb=ZO9wh&jfqWfUx!P7C&nr)uXYY#jxRI+{x-xu0l((IgRywtN#9 zh$(EmnN_-xgel`eZH(qPw^{>l03+51grVsBNi-ix_i=@)0FXiU@Or@#YPjS>GnFB` z>3?A6dJOqVgrR$g0IPXAk?@}Vpfi+`reFdc^NKd=My%$3!W$^M$rV*u3+pxdizTOq z5tM7vG=)zhmKOIl{q8Q1_6$Es_)p1e{u*3?BxL@Y;dTHp?;se`T{xyL4t?2N`|#yH zAUOGYk9fT-Wu?9TNjUjRBe+{1bK(M^kuLT&l- zrM*qpb3VDV7z^Wdizc`GVF4ZqFJ+J1rAUM3L2x4T77Z{vxB$Ins>-z>g*oKzpA-QG z%_BU7JK0KT?cCnvCnw_;H6O~u-}MxE>ih8crLF9CNvX~3^j6Wyj09~P8*tx%$8aZS;;kBb_073rphIap$_jXyfE%C@;q%X zm3{PLn3E1+ELrM$ndN9$hylZ_;*B?r`;;_QN$+CyG@%dhUspf2)88L^$7dN{GoVj8B(JukRwvorzqVgo*Wh+olp%M# zCMh@sY+#e*X>zxdg#P4YEPrGOPlbi_nQ={jfl#ym^5!r9%h(t#C~&0*RdB$*1#VqiP;psT?`4|7N#e8Q?pCgp5QO^2&qZ zbghKS{vkxM%bo5=y1cq&RzWZpJ#;BK!WdUNwYI}F{?e;$eQx6^li0O)2Ll6{()oNV z;V`+`*;x;(&5HApcHGkE2U0X6jXmWq5}1TE(X|!06rJTUMyFHdLS!qWMZU`=Tu~Uj zyGGeH`r|75Z*7Q1XKAUVo^_~Ad(e-gAE|!=S(SNOqnl0FXR=w0w5ia~m~=&6 zR+>~fo%@9lmlzM2g(cI_J3Q?4>7Epc?76L2PITZH#UYy@8MOYjsH*g+S15nW{atW_ zoU~jxd-tx#QZz0()0f2)(8TnSTfG z%+ed=fqTVzWUIv~x11!HS+DQ<7PXY+L2CqmrwqN^bM^6!q6=(_$gtk0rJJ-H zP=m7R%DUGe>_lbHnS+KWw9Wa0#Z%EF{u2MwgT=MP#+hDHL2@BLzK>w62 zuEsLJH1)GyU(m`0uTzvhWp!Vfm3wF&Yu9pgJNlyAlL3&>8l|jJH)rR9=)lsg%DJRU z+hmoiz)!*phn=lRcrskhv#;xU4she39&x5KU)3b@_8NwIcGAB}0Q~ByQWo7PU=^{> z^od;UL4Tp&`FYm({PK?B?pDB-)+b6mn5I^&PiK;05e|m+30;EA22*!RI}Yy098)(o z3@nV5F=2+y0lj#WB`%Kq3s;!T2OthY!jb;YBZ`Lv%1GvR6t$(t!7-B>R$QsRNe}3_ z@eGduIhTWuXWkl@0?Lg0LCcrSa|PHGq4kFm%S=2!Jc2)oSR~pfg#6=eAwZ~{Ze&bt zTxE0Wm57-i7n-UP^Vy3eU(E}addwl8oOI1)hCCS3XLJj{s$vu%J8?%_&nKKF5p-2F z_Jp=*n%sTGW%cppr1_|ZGWFHVBaB8EjbI$__`}D$K?ycxV$56>nVpsQ)=NY5#9t^Y zmlr<-sW;d`&v&CqB=Wz?(vD?#K;u{1!o>Bwx*zH*G<^R4X6$0|P4)t3)g*7Aq@ozcJ9bJzWzVEnblBrJ| zeOc>3MHCp$vp);yC{$ariOgIS%(9ItxrN-$AESp|v}i^EdH6Wh1+HMEUP$U$QXY0| zr_iw^LWE!d)ksLv4 zQDud1HBLNoY|#Gn!Sk$=Rf}D<>JAzQdZ}A?57RcZdkbTtveS>%18QFCQfoPHJn1>= z%aUY`Y;CSdIjqD?u!t{8<9@2SnD{6-Tu*(5i(>FyX3+dsY%*DaAV-+-a2fH{+*X5; z?6ZZE*>TQ?Or{g55T^+_&N6fPxh-7X7{RH7W}mEye&b2;m6il$#F}9H4Bg4vVN<;9 z=YEUFv3%Yk+N|rOp$yg&HO+Rlv=zPKWWg*fncKbaqbkbeYc7^$V%w&-2o5%>0xB%r z%QJ1Jn3bMeP1H(XfZ{8W36psaTH4|zU8(jJ*ShA+U zeMfScAt#~pMV(@u={60Exd?czJzybHj__68;^`#nLbqn%Mn#P?~-~jm|16`0Pcx7 zBq4VRX(;6beSi;ZY`DmK!Y+jDKB`(h-XJj?_W-gkdh2QEgmZSyDF1+ARLHJ+rGDWz z^;&g?M@%DzNez93)FfWqlNPQq!>+r_8%NQ4RKrK{=!<$=jr0Ru+zyPVx`M1xT?!jh z#M>6{doW;;5#a=sc^*eid z(xo*oHd!tU&4 zcTE2WSLp{XpFXBGrsf}+rn_(54#ndo_F6 zsL;gEe3&%p&1W_HjpBxrF$-3xkJrGL^!jLv?imt2m$LKyfJXO%0Sno9@VJu|+2g5` z-Hp?Y@fWd@6Ga58lhO%UHEE&GrFF}+Pc9b>Ma7k>14B6af)1;$R$O!;u@5Le)$*gT zj8Dj{D|;P4$20S5X0~W~Is+GJPrj>gwvl{$uW*=VbsHD4uaEUe&oxP$b@eRW!umb$ z<+*#`n&)Qb^Awk&h}Vc)k46qC%`iend_m?JWbXUlmG8wTtg_3$U_6Qw;F+ZMzEXj8 zk~$Qg_&?~IqvfGB&P6R>E>WOmoH^H1SQrn^y(gemDPqsUYP35ms`z5@0KL+33@Yt) z(Sg2u486-`9Um1TwfhAP+Cf+KP#qC!to=ox{24zVdzPqw$mru znvGJrdM}?>R>hP3Wd}zA*o<@&b zxUM+gV7|j<))s5lv|Q_SfdUYcYk^`X9ZHK2jb!_38fW?Szh2^bMIL5vH+>m0#3I{) z0JYN)PpT&95;S16vMPhC8NRamY++m(dyodpw_i1>xp(H7T-7lkMQ4&}qOKIfZI!cB zRanDO(7K_!i|&kzXm>le#R4#mXI8vhiaiAAec~JPzp3w3;Dh^4<`M%h#YM1MB(uz; zS7N5U8;@;L(ghJVzIZpp)9@bQw-;NVKw8dX5$Jm>&848vmbwn_$L9uWC<<@ab8BoFK3?x(TMmg2w*p3P?M*Cx@};&D=pgW>3$+K#!} z?2fMuQ^78&HLngnL!s-O+|*N-26>)#Cug+*PW58Rgw&VThv-mnvI#?Pap$yVL-5)g zIiiDnla!BQUfRqrbM9IL}~gCwE@WEAtn zo215GM5vy`UrMzp6W=w75lN&_);l8CSsl)iX-nMc|o~i@8Ui`9{)_5LMKKRKtuRa7_Y1 z$`r{5z^ZcoKiFtNm#)@9GI^ z7U7{?gRhxy?Ghr&E9LK0-^1dp+ts)oqDiRrIKJy%#dBmfZ;mMU z;vZ3nA3Xx8VG`neR_$rb%_HhT?kY1+JNgaw9Js$JvZ^folTHN=(hxQ{+Dp9N9PAJ-9*W*YTe&sL3ZRY){AI`hy7mqLKdfFkg%>pjz=~T(b&tJ(T3%2H^ zM0+rNs_DUK(D_gncbwoxOMfyksD0>s7^lEclt!kX7)VzYFuTD^MvQP{mo9-V({$cTpGrj4%I0Vkwl+= zS+}J63$c=QUXPK92a1#jU13dcjo(X8hv;NDMP@gkdCUIoy=y>1u17f{^_qcRwiybg1{2}x)r$9h{#k~_a2 z!79_dewHpswCCdKz?GG|bEntb2N$J`_&y4Ze%4GQIl~9fwViak0|S%SxGP&sJjYyc zjs)4ti=}0H3zlc#Ed(*T1oq1Y8eD^Xc2krGZx8<#oL3G7)AwD8itQY1ZM7}yyf~Vj zvV*;!hN~MqKhyEW*c83ex%$v<;#T;;=zIP#df4$8W$Z$sjCGia_u)qwjvBDQxqWRE z-z4lX!DZ{cc(S*Ym-mwAtOcImStZZKdv~w9M^2}jd){35N93!UFrkSx9$D7m?ktYIzpPre zK#r3?jH*N+0M3f5RWi+2&mFD4#Y7dCE`hgEVV)IVcAJo*Z6;>!Ck;^_^w`vPF#|q? z8{kqcd|I=Po^CjPcV@75(0^c2z>S<_ugtgC=dQa)Zq9JWTPQ4a$%}?(DB8W984?-z z3Z(VTb|i+%ppj7K4Ls(|^XUKx+5+9T&N zy(B?4nU#VHo)mB+On*B8?;8AzlO_W3xlFK!#Lb@M2ow9SE?+7-1UFA3jjt{%v2B*Q zoQ1IDNV_>;728T^)VmKAc;q4YOW9*w2#7o)zdw*J!rPrd52| zs461{6%sJtZj($_BV0Lt1SZQdvTAPp(d~TLROwo#RrK- z)?4%`us_7&v3&F(ol|St(l`M%i`+ZGQ6SbTXAMvuEu$3k(#6YyZA(Z87nGynm}(iy zP@nFFB_N#1! zUh#OSU#0doT=tXK!gv`F=%qOe*j>WPnLm|e7h!OC7UT4r zml#eaZ(v=TP-z+9ae2`1zxMGUg!*~N( zk`=H-CK$mOo7rOdYJH5^C7Nu%fO8vD+`$2k>t0!V>9l@)0sDy-!jxaPz$`DytjC^n z-ngR{k5<6coqX7@~vF)0;8DSEW|JeN-sJe+N^;-p$(M5h9r z2BH#(_loxjHNKmdi|e|>I?+-Nf=)^=eTmlSHQc=<8;KsF+kcQd*q^_>1$gLo?1d@9 z&@%n36Zi^(;Fp4@{blgC!P?^%EGOu7N6yeFqBrgxdTI;k7lt$8ODUMt0OuAXu%={T zKCXBsx~SWFsdS=XA7#1m1N(9+K`YsJAvki!6e0}M z3AdF6U5wSP4_jTQpDk(_a@4AdmfsN_rg2KWrK|Fl6VG&|AwKgKjN$V$JFZ$EGwV_` z*oquDJ26Ic%1#^Lr?;#2$|56P7m74CUbS)8sI$uNdA&&`CB3kQv9_KdQ7IX{De%Z& z?G3GN>WQLt?aTM)Ck?kniycNgA8HlpCy`F+R>mcSCLstMy4Bwe179?*8T>2Ep{@2u!vsSyr}mJ2e9k<3yd65Z3{ zGm5Aj59)r#ZQLvG`rSJ*9Kww_RGEwYs?w$xvB ztj)*k6KIg>O^gqc7!nn&(9+Awe+)ih?oGTeN0IFnU7@>HEyyP@t^5&{q619eU8X^WpF zkjFa=>(;6;snkKesHbc=IX7zcI_=Driq}lN)ZDqvUU6?O%nDpgOluaiZ*>e9yPbor zU&Cg?Jik6DM%KW34>zEX^aWpg;VmWiexvukQezERH<3fbnb#v6v5Lw(CDaEQEiq8H z38w$%6Q16l05JpiLmAe``S~ZpiH8^CR%C<`;BMFS==57NkEXuGx_5`wrO@^?nf($4g}VM8|UBtF7MA_|~))RL3H1{_e$@ z-e!IQtV+lE4f|0REKur6Z}{)(Lc~*U z2ONF~9mx91-PQK=jT)Sc0?B%}5VNBWA99_;N9t9N@3ao)&pr9$&;argZJ9mvthiS9 zaxK^UI?Yft;NwmxFc^wk60 zq?)J2qRURPHXJtZ^}M&7>5lyK;k!8V0>+#&T9GE*0v4l9Ev-RA;rENu2ApMCCF?gR zzK~eRePZc})C z-Upn!g2e)i7cz2DnDkl!A@UjqIK4gW8{+{b%3XNT3 zkvx!u;t^wCwnNx_*wS3A6b1qaogma5+oItF|a!VyxOaHEFHgnD4SWD zMfgF1UhSq(*z4C@UfMeN2QS&V4DP;Kwv~~+mlY~O9iP#Wd){%b-#2ugQzHc&jxXz= z3)Zh9pEO%AR^N*e>|ql0k0+$>-SzEk)ofOrT(h{09hWIR5NtdNyf$w(JukP`x5s>t ze$zidl#&A;y663^30=g7!-)}yS{JOeG`g&q9(@@W-QyJtja(AA>@}=u=$r8oV2gVh zU1N6|CYe}BCb$lZsOLC${zS?fd`%%`wlD=qBMrH~!muYfXsxXuw!$1hH{Z0t)U z=67ta7qeDT+D<8$Cvc7r#mpdB*$_2ptcHEVOB)Cr?~HJ=_*Slo|Ll=>kR)+(a)iNj zz0{P&ws|gU_p)$Mw`G5|*R9BA_AWF>O14{mP3my=z)y=7N8N%SqyW!X=6FV~8P zF-GE#L`?EV^*q|*6?>q>#lmz=j_Mx{$~7Muz2$>mmiYm}=EXuC`bLix?!F9__p8{{ z++GuQt-3KLC*!YS^8t4$xi$x;jGgLX+zJE*x+@iH#=)ihK=>PGLcPgm5L6M^1AQuG=MPLjUCmn@7@>obm4IR6lYn#Ips#>ckitraqBr|}Kr zDy;_vobc1ki&7~{fook^hR8ZIdBlSbF{)M~*^h(XE1LJ;pHI*x)=PLGw(piOaMF*> z<6Gx8&6MA{#R~rzW_tIGdF9CyEp+Dq$`DxM)A!qGOgQ8%4M!owbb^F)-Z}m5dY&}i ziQ8f$5u|U9!wf2G?)Yincl+q)DHFMH_i1B62oIu);u^)-dFdWZ*Jya76ocEbA~PW8 zkcemAZEeRgRBBD?LTIQCNBe|-H=d)+3oEH1_CW6KiZq~{1i$Jq$?S{MDF#k`No{_* zk*PMmm4#GRJa9zch4R4No&ib*zG~~$rofMqqx?Ak{)u$eKx*97iGELA8uu&eKWUvC zfyd}{(y%!C%Kd^ORBlB!CKP%VeZD6yh?}JG8G61P|ESP|v{>e3gc4k=?;1Pb;q4@5wKp)r+=(`9n?=bAHRuwHuxnOpvMz{6Y7#8M=k|2ew5?>RGHbvd zT`jW>9~Kpdo)4(ACPf#gLfkJTTnkT(uEu4~HXYOR4;6g%M~(9eyrD;x`0FTH(-lv* zxlxS0OLSp-jlJrEV@$*;m$y&Hfy9#v;kTy~K_REdOWsqFY62;XWten%a^S|9qz-F8 zZ49}Ci5UF&dWO?E6UNa4&u4zd#gFoiZin9E<;WBYeBKE}FBXKbF8Z!UW#<+cZJ?_! z3m#(Vl{-frEL*+;aq2qeI$8{HjOe_rZ{Y8p857ES(6L;6{1Dr*>5d8pd5d8R>9>aB z`3!O(>I22x`jcKyk&t5_5U~~K=^oiK#_ylYMs77H*9jHl>T*2)7}Y3T+-kaso@Cdo z2Ar(w?|SaPN3T8FuiYsHY#96E_Qxf_k(h>={aaH3rR=n`L7E;v$F;}t3F+1bX-e#^ zM((oq_DTKc%(%UMFBal)hjiWI2+KI%-KUA)A`MG}L=+Y7&Ik8Ub#~AvWHil73_Y^e z5k1R?f5qpJzH%gk8Aiy;uIv;oeTMNqgU9LJ-ow#qhq}3(yBm5|xZCC_<4?unPvdz- zLYwva9=*9bnCUTEETu@}R%N>i8t})l5je1PE|k_cGxr_l1!n=RIgh)LzW zh&6#K^usD5=W8WZ!J@vjZuDZfxdbuq+rkYg?9b;V>2_+mMfx&?Zn3JIo#+JhI+rgG zevcPaF&XSl75z{nmgQ-gIP~$Ha`ZXY{#XLCZTYfOVUjy~;mY8_*MW(*qXOB3yA`2S z-n-Iz`5p|gi@MX9vIvb2La}SIVd5k__@_b+vgsrDNw}TAt-1$ti}T<&Q1ZdFAAiDw zmZ_-StvT#n_d9bGv zD8qIDF-Z=q7~k_$(RXD>T{aEts^e$uU-ifA%Iz!HPxou7=zg08)mHbpHgNVcy;c9lt;+YZjsN8Z{V<;VP| zUCfV?#yeGRAucZz)|J(u)Ti$WxtG0CK2bHPM>t|aO-DcJS+)yvYkeC1K=*-6UrxPJ zAIXOMS6wo-yF`zHlNzN4_0qO)cCMR6@a`9*bkQ2kx7bHtI=wd*2>Gzx1m8H>*P<+! zY3AGX$`JqETSs%A#puj9npS%Uy6zMlhuhuof8{7(eMjJRPCOe|NkM%ja@?Ng{*RE3 zx=yW;{Hy>ylOiB-_`bn}nrCyhq*wS|HfjPR!A~+(dX3LQSIm76IIAK{b-i%pP2#jW z>mK`{Eo9^mf+})!8DS+u{&h#Ng@=csVF_hhPv_oHcx?4q|T&AyV34W zBI80cV>WNMY>n~wz1$*@e|L7yr<99Ia0cKRcdBHImJwjMy*;%?KNB!m@ywP>njd4H zGgt}Zy7w1%89aS{d45v*{L`ygu6vl*32Xl`qalIU)*-p_r&Hhn{iI-LS*sWuZYLM_ z0Uf}7Ad;?Rvqi4``Ze$ID9|&w+)M;&Vg(wJ(L5}UWk>oB90%W)OlRB@0^CGDidmonNrMIP{tT-$!d~`Ii2xq zx(*5`%;1uqFV#?d#WaH4FWp0bx1(sfMwm99o0RY@OBUEFC>pUkUah@sv_MF>^&Ta? zCI}r?=%n5HxeZ&wKQu;5OqcTN}}&~>o8-g0CV*Zjp^ zHqYzAUwS>^ePfS|>_IJcV=K9u9+2&@&%|v!(Fq^9y_&Q#FhOwQS(wgRn}AI{7VUll zgwKFbd#I!N=$L28+}b%mk}6BVadq|X!F6trlSvqkzBD{ehKmy&a)jJcF% zF-jWM&Y5>M`Z~>qTp8EQWYjFjW=f#Q=GSiVZf~LBwtA7x{>0izMU1r-0?DZ1bo9|b zB>Cg1gd$9tO5wC}0~B+#MiPB%fM%F;bBxC1b3ae;bt(kAIFPhO(OogsKGh;ln)ZOM zK3hDmrD%lh9fEM(5z7Eq7Z%ZlKI2y0{It@&?kITU*TNy~TOZ(kJdHIolSJQ?nr_h2 zuu2Xlb)c}Kk@F6?2!u_DahJAFbm=pCd#&NuIDAv-A4R0S&UvC(gSFL1(aj%AE_pmU z+|b!#Tw!gZLrEhO_B!e>vmc;HpWlV|&6L{^QFEfq}g@HIQlig2>IBI9_<1eZbE_6#g2oSU=s@AV2&TXVXi6SKo3WT$@ZGi)tDUxqlR zyE=zVSZ2N5Rb%r{6+&|Ft0%c~yRtf)O`W!A0yEC{XjObR9KX)GjcO<91cLk6M9dME z4C`CG9_k^4bUQgqYr#M=YdW7c89$B%Ec)xgBMHw)I4y^G!0!;ih^b_+wzykN>4Zj) zVZKg1HdnGaxH7OiR{pHe?<}y$YOLcVYh)N(@JQD+iEsB@X;W6SIHo|0>@+85!eBQ! zBm4E`-SNP~PPhL`j32TC(uT+7c3$n#V3|9g@Q9*WVc5FIrCts@#IuhQ00UMhkOWC< z=gfMcjWt;_?c(JpwH_hcOx`;w5_j zL$g+Cj8@1UVx)4!`|k zOuLG2eE7;1&nm?O^gXO_)M^!P^DvuhVdMVDt@2zy)xxUn3r$2C+&J@7pl>v9+qmST@5 zm<%!sMr*%V%)eIh)X8#{xwN15-CZ4g_M+oeO$ZsAhn2f^wH3Wy{w>`pj;fmTqob{R z{JPmJ-7-oQBjlEPKmy{xY8YePT3~*mSv?zpKc#4K&hmTxcvj zYeC4;>kG+tut0(C?2K5isk4pBv7>8C7MnZxTZ8>0V`-*~XPnNsS6eAUE^|PHA8((- z^7qWR|D3^mLd8Nv?G74amj^SxTWeNf<37AL4mHf*pQ&+WL6>MB8AH&dfE5Yf0p!{k^`je1t4bft_XX5squPgJ|Mend>`41 zq4IvhL;L#QJ3$d-{O}R3rmV&#>$1AYhh!Ea^^~a{)QTh0DNPV4Dv^E&oN@A@Db0_a zbKstMZRU6qk?gM3sYPvF>`=7&n=;yem{U18Kq7L~bL;oXkG&hu1XlWZdeWnWDRcL( zQ3nimu62xRRqvtfGCSI{MRd)eyd?T0^;Bw(mP zQ1GaEl~kwlc~C1l^;&1qy9ic!zLwdmb|-S<2Klu!Ujuak07VDQEB6nA_r0q(FpJ^* zDoWGH%)d4fN>#DLNaagFX#Wq=?Zs~Ftlqa&?K}jIE#&F+m`H2`D z2u5j_ku2<(R9k6~ zdX`dYKgJT}9#vY#3RF%}LaNQjYzDVf#od&Z5ptCj9u@p3Zu%|$jMC^MhjGv}a20Xo zxzrWg9rgok)9 z-G}d-^we`l097ogJ3A_+(|IH9qh+MoT_|DAtx~ZF`Ls==w-Hc8-T?x9j}QMu^i71G zIyHT1nP=tpLI=J8tmG&2{=bw2VnJdP_ZRppz|_YG1R0lHF*~~xnVXUI!VIaN@lxwQZDTR}Su$F$}VD!DV zBFZ#Q_K&d3KO~k~dTk!iR5wjw=2>wiiw^^+&~ zhit;a2HKF$vh||=YB~Kf97+X%MERh8;$O7(ue2fZ8ffFsh2eM;TK`_tQy^QM{Qne) z-XP>xT9&TvTisV@$^QE;09eE4+9fRrsO_Yqgy@FtfTK|M%^o!b8LHUBts`DTp-JKS zLX}XMtyd3_sg_}KV+Bseb*b-T#oDKWKhW`%no0k(9IJA10#69bgdqBlv~B>0BK|&d zkvQGCilp;QE6Bh~A0@(6Ek-}PUU+e+og z0ICyPGMZKk6AI_-7vTFiuZueM1yzCCT`J?-f6xv<03Wb=>)ssf>}TVmzI~0r_RR_} z4s<;h(^lXjHW<`_$o5eOA0Er-F$#_m31)53pIkBM2(QNUV(0BaDNrF9-~DOln#B|z z_g3lL&QDQEnhO9)D0}cNliz?VA004bhRVSgHg}Sjp=w+sK*)4M&ujRXHk}@c*nH+NCE*_!`@88s z-E+(kigG`)-G3?s2kAfuNewrgn;#QFd-qjK2?YR5EHj{R6Cku28@6;X>Bf04$*)U+ zv=IPR9qU_US13TeS~!)wl@M0u45`MvCusS)2pOxODUUv39!NW|NB4P0(MI%y^x!us zY+m1#%^KkbLQw#>cuc*UKSi zp^x2LSFhvZ=4W)94X7Gcpx^F)<^`lBfzbnHhtQfQ&fYiZ`DAidUgK&RjjKU*!KWoy%dU3q{eF;Xb_1MQpLi55*38%%O2Vk2p%Fg7>~@2h*W%L@O|O-akdn<$iP66UZR znQ_spy_|24^jiCf1CN*pl2d7=Mr@N}W&0E*DTP`Q}WYP2!D+G9V3oK>lRt z=)WQCZz6(hqbswawG7>Q1!6!=#zv6?92HyL!J9q+utKqGxbpN5M|Z|RGphVGKYGp-W*8o-}D{w~0d$VXz4gw1aLDnbhICDI=rW^H~H^%S>Ih4hK%Mi#19pOSq zZk`1z#~hw$sIScAS@GoNXqh-0I3hBVEcg-Up=P7qoS4~Ic`nTU9#B_X2KTR8;TpI4 zxYgUA(TT=&t8G8vBbxQ4y}9#{t*rzTfH}w(KC2mRA~616(0naku|xrlp0@m=Pvuv( z{AJN7+5iY!-hx4Ds3jg|4o9g05r0B7N;?`Tsr_2&)>Htf?^PuhW4))y?&?6vzQ9 zBv9_2j_6O@{cUZKCV(>#twpcnvj5h~pA_&R6dX&JY5&LW5gCBVZ0XGZ7ZC9PCpI80 zQdI4K?cNrc45IaW+VwAMOOpVwikI}H_NH4C{zebv+PeI|lY}Ht>S=$cpfu6vJF{O( zjk_KeZxRc7O=wmcd_f!isP79q4I$!{YV^9O|0RA3F}=PTDcB@f;8$B8AEn%8T6ghV z++0~SvamKh_L0=aScQXkr^QbclBVgi@-gjnzt8ojF98!$14K_Rt66UPy}rM-f3Yh; zT1@P1Xa(R9{+H@Nr`NZT0)Kh?Sl&`o`e>E7vBZ!(6 z<`gtEG+yJ+H0|@x2?TBTrBt@cjfd)ePOeS>XJF0dq1%% z)&>Gd&iEK6Ra%YnF#O~gfM1|2n}(*P zkrsC=vq`mB+yaFDe!XrD0R{4`)o7?4u;rMG=v9xJDy>0|<_O)#D=zx(iS{DOy2GBPtI zJEQ6OormjP9VDMUJ5vAp=qI=vNCI$wk|Y~~1d&2p77RQfY5nwm4>b{SK)~rAf%KH> z`PF#vBh9BK($asA4TlhZeR%^*4?6T|>KNks71JRs{M$E~_O7l2wKqRy zARr)oQs3<-9~(?@O>a11UD*EY5CD~u&d$zYzb{F?@XQLRe^aGDH9Rc%$vHiNLiU^P zf3Wdd67*PRqyGuu4wU#@HdlZA*t>)<>(il6Z}>z9sR{4R-zIo`dmG?+~-g zAnS3|OC5?~Wk|)2q9XKBZTE?jv=0|*c{-m@xo<^fWRRz<*1w+=qo%Xl!{S}4zfUjO-6e;eDD9p!2>yhsAr6-9;snTXbRbPizxe;tIDUv%K zra_)}CDVHgkyQM7Pspp?ZZ?~7MdZ+j4Bv@~K0B%}%+~`-)CE2Xr&A4!oRJZzJLA3g zSArG-PW45KH);-X6shp@m&FbCKqVn*I_v&5^ZE;2`zf@zSI-EE4J8L*@Hj{yr*H_i_c5Ji{%<;X$e=~5w!*Y`;wyfp4_iATq7317og z|8cD!WSOANY_c z(c%Bc)>j6_wQXBB&^QEl39i8{xCD2CJHegcE)4_-F2OapySux)yK8VKU+3I+zkAMm zw^kLssiOI@)?8D@m~-sE%n7v&Qg_rzL#xrE^EV#SK_AO+Pg90J_eiJ~e96Dxu}S_$ zgRAj-zEMAih+)1+0K)|TiGu%mBI$=_mukiyLqsnZL8Z{Ct_>3khhw*#m2_}$D9y1) z3F#f&vd8_OE&Y#o{p$&}2NIse%bE}$5-w0uNW!moU_eGqO)W$#7w>Ow@9&|EUp^g+!Sn_Lgirrf=x%P&=%6zlTIZVM*!5vksmJ)2Nl!X` z^?xnDK$fV5ci@p{&GY6EHROQ_#sJL0lU224Crg}NH5CZ}NCS&zRnvfI`RZ|`)Sw>b zlkL+K3N99U33b}ee<_=?WwW#bNO&Lv0g_JMhM%WDQx<6M93Hc`fu&a};Tzjgno$lZ z>9Ie=Id(-SJM2IS!mS8c;Qrfx{{4(VLeSfgWw^&4vLGx=rR6^oSjFl>P^8>V5oMd4 zk4DA!I+d>fWurlke-yZus@w?<{yJx{9ws|#_BSOT78gVA|00S>A`M;40M`v&P4n72 zY4gWMqx|Hf@^XZ0PZw*)JobP8&k2(7K^BY*@18EzY+0nK{mq<0&_I@B@aW`0gn#fg zKR@t1qsZMU4RelgeJsh^*tf)gnU9M2rl>`cSCd!KUpyj_0LAk=qSziv2}(vrdHh@x z$)0c|i)9*Z#-la;<$cbI(eg#?|LhAW3B53Z1kM$9VAd=o;P0FZh#4Le+}-s@iUw~4 zd#DP7y7lg}sdxY;@v0ran$6k=@F-eb{aDIKT@PadM0$(IoEgO5vN;a9EG>T$4&>yW zf0il!Sc`QHl0LtO9jgyIeWK*zj%QiO1zGciJ%R|iU1YBi5p!>5i>`PGTx}D7d+ty+ z`SeQup{YKXsFhtg|8c}%)_@1}5C|-aQ<1LbaDzs-q?XKwcd0#&rY7=mjuDr0EImRz^OP&p)t){v(?BWX{WXo4Hz~`| zYI?*~?R-YX?X?-5=7Q+3(s1w>zbs_%r#N-Y#{S!`f^%d_NNR=n;SK6mJz~IA%zKom z4&~D9r+W0a;%z+hsJh3Y7@p~H&?0jnz3MeRt z=sVup)QO%U)88znUn-1`yWOdNdVU``Uf!2KBwh0uj>kcYOW>B{n_t>!WqmYyoUO%B zrQ%e=h(5rD8eI_#){o~i!IA;e5L7ZZ>u7s zV?qW2x}l;WwJcGnb+d_#nCZAYwe#rgbkEWD`e6Y!!+){co3yBl`aT&ezux`LYlh$y z+AxYxqy*-8#E?un?2l^F@_S3mw=l{lA(w?=z}J=0(nH!vjMlE@dAOP`SvZ2WTew?K zlHMqKsex?#jdib8an$_F@Qv?P!g)a7tkmQ0-3=1Y=gk_ugrk|tarsmPGT26*K3MwS zs%mpd-E*F`YSHp4Qx%LJsD?cs%#~!@fUHuL3p(18z!OeyE@Sl!R|5p!Zw(8E>V;t9ACN4s2YHRwVwou==_{3>13#QuPjmPZyn$%=jvf6q{e>z#p4= zau~IgDW5nOrY?E(JbiDJT-p1~fv{c6EL|6jzvWe^w#OR>-3{{IzrmN>k5 zlmG8lZ}--Nv>FqbyH{U3C3;@l)FSwr_30DC#Qlw@QpzGiqfVU6ATWQ}etA zTUt1Sj^Li>TC4TXYTg8d8nvc%MyM3g^>oT+ef!8)!Ri?e{w&M#e`bZj(_*=K?(>Kq zBV1*P63aQ+N0GU?HoH7Y?RBnF5w1v#(L@#njR+ z;OH<0pbLfnp6@>4biJOOrjTu)S4qN!kY<^BAo7Q#$GqzdP>aj?Gvj)EPYPgSybfvu2D(=y&tNt89y zQ)20Q``EI@$SGmdb2UrsN~My}*`c`ohTbU+x4Vd5lE*n@`5f+VoH$5m*t>XzZ+Wnb zd8&GOQ8i!0P2CAv?o?kfY5xSdxc9Bf#DG81w`pvb-W~<$!a~krOT?{3?0GLY6@#P2 zUgg=qX+z1t-PHP9o{7Ek&&)uP+1v9h8U>G5Pm#`S3s~Rk5p%?~3BK{rNl)_#PhNE- zkwLSeANYA3nuVi55(h&g?~jK*hr~l>&Aa#|ObiiC9{g^Ey22Y(2hjVnZj}*LdKzS^ zGGt(}vBp-o-L4=PSuQmvhq;80g>EJuv?$SLsVG`7S)92hggur-Zv`0GC$@VAB zDEtoISj%nAgQS1wbs$_EFqVIu6oZ2g;C8zy#3bg~ybrNTK61j(FOX5QOlkT^aId=# zbW|b}U<9}0D~&XlcvUog_2|}_y;WzGMa=^Z?V;K&Po1&TWG51$%e{kaQ~c>Y%tZHN z$229}(aNh~zFvy$#LzypsZ-{=HyG02q~mdry!KG3Z-fF(CFmrnvhzZv<&y^4FVfA1DP*PC_*|V2p2W#oC_Tb&Y5LD1H_$LGlOR#O>Dpa0tYp z+Imc*t&s&7n>|*C>dnfRGTahyM7yl* z@kAuZC+2aB)S}Q@6vf8I?6Xj+7rXm*?U7SZ@~bBJG)-Fe%u)SCcgca#{?Bpaoc~eW zon;2iP%U)*phVnLxM)ASAB>aD-g)j1kH5r;xY4vb%16H1rG7uuOmu)Q`CR1GlhjaEfnhWGlN=&xt)FPV#nYsE8M0$P~PoyR-xsfFI}({LTBU7+VMJ z0KIiJ;47~sjOs18Ci{Jl3EZbElu)W~WK-;_5%nuBr+2RX^-=UlD%r*%?SZc4E`uQ1 zadr0D`qE2(F4u8{6M`sn0!9N|u!31bz^CAh^pTbMR6@>j9cpbgyXdRlgFUM@{XH?6KA- zteuXw9EFQtGfe~>@bPbwM^T{yk|J0X%ya)X{Xbv(33)En6Bl>!4DkNurIP9M#9t(>Ex4l;(w@FTCo6;*{K=*-F=LJ-)kaAi8 zKzm086#n?DQZxoq!)DAzdayWqxA=!Xqu7w4f<;SWD*M>K{>eXoibumj3NPhGXyLar zTM6z66Wo%F(On6!F|L=$4T$gte>dY1T9}=uiy{5VxDfFTmFK9kmF3O~OVW%ZgdVjw z24N)iO^#p(Vh6N!0zuU&-YY&N8V@BzOo1^&prkXZUB&k6+H^O%MPXTDLJ-xEPTeBRY_y9N zM0-tjAQt^U_F-#*YWO`K4ufN&sUdz2Ofw53aWyxM{O)^Jh`|F~Hgb8>TsN@*JHk6} zc53>7G{@BOpznL;@aSvfDe-Q9-|vc1=?^fZ9w&&CBGKpa~rmhxHL$>z1O zz}UZ+*z~_qUjX>kHZ;onAtN9vW%VLW>c>cj`Gw5=Fx3$LZhXN_QHRAL;vM6ISIoDZ z^*+||Fc9)b(}gkHPy6Q6DvrcmaWMA`STmz35Mtpzk@WSh(86tRk<8%q2*RMV#LOVmY8)sV;3ZdWQb1kYe)Q;-XRMF3^G5vztm-EGBu;99HCBN^%`D+wa z5+>d{Yv;FJ`dxNT-T>kZF+Q8o00N(g)Qnz&pDN}>mr}WGB@6PdXj@08x6uJ_@$rmVdP`UQJfxGrv6x#&>M8K`Kto&B|&V6!ph;$O8fEB1V z^1A>}-}i&-RkN3ui5UqG$=_X8c1R=`#r1v=<_*elZm#T+jjm2xN`9PDl|-^fX&{wF;4*j7+t6ruzG%D7Sw>A zW=cqqRDI?6j)1J(BDz_$t(E@7$6#HY+q?cA?v^=f#(D+pFtU7!4XlG%(j%Cx#jCAQ z0NtK+!r!b^owH&s6KCCzIJHV!T+fGfsk`@b+fwRCa@21JZtLF_F_!#D3s{ow3GfS{ zH|nQ;g$rOAGNfNhPJ& zOl}$UC_%}^HLDjF8tYILWQ$OOt|8a6;R3nsG7FsnRHSES*Vz7vLl_>4Z$!QVb)b+Q zpVmVik5R40blyxIZk>p9oWn&D(lf!fy)7>`S4xx?8 zxNTMC!msJPF^Htjmft@Bl2|`oidPe+^b_n~>-%boM~Hue6GEp?+-2}1iCDnT%`cUv z3#|}QDG}Eq(B8%FEjw=^tUd@CrT#z10`_mhhoB{RDNfYg-aN#s?&XNrb(v$WT2~hf z|6jVsF5Tfi_j`?osy!S7nr-c{+CtIN0OWwa!!Tq-oRJC_??RXF@$Z>z++@84fF2)g zP6*AbZ@kzzh?$9T+le<{NwK^4M&9ewV>COnyVc67i|Q#aHHvsIHZ5gqDln@NiKII@k*EXog~V1ms0c= zjsrGnVoh;Wbc*eIYb9$q9DSOBevx7Mg+Zt_%})ERkgBtzXfFyT7T?l>f~1w(9Q`_K zaxefI1?-&?dz2=H*+bt(;>lSr?W^FzyPaW zm_;AhNc_ejf&wB~k+3m;@VLH&r}b@4FE3ha_KZoC4upZyqD#+{YK#S7+U@JHUS852 zdm>Yr^~gHi2RfF*_KStxn-#|yJrcixzkdv7}R^|rI7WMQa0Zs3Pr-`*7~cZ{!| zg~rW+n{S&{i%s1(wiyack+pGIE9I}6R>7`M-&7a)0`Tq8_-9e~oH^)55*DTmn5T8x znmvQ3b$`NyzzEYM$mhrpq_`u(^pvM~GlP%L4eV_Xqnj*QV$}nz;Y)x*<=A>L7@qG+ zb{E_GOgzrr$0lDJ{WiI~P_V2eRtMuB<&sN_l_aN#`T6@9GkTpR%f#|ygbuxX3n(CW z8$$xuL=+u_W!I!SFm3Eax@`cE_g<2u@w^yLQVVlmrtBQ*1bdtXd`EhCQ6y=BN9lO< z?ivsx@13(e2DGYX=2we>=jml;`^a)AD&(YXwUwfVX1y}(wdwqL6`b-ocG#dVqWT(cY75eNzj?wcTO_gkr^74nZAg-}L`bXgXuejiL2e^gI{`ARY?i8sZVqmL@p6UaVJt)KA+YSQh zpyhFC~77QZC zFpQ`Tr#`!s1UxTAqEHx#SFbn@C*#n(c)v-bV9n3aFMPDh--k9xUAAn?YBF3I!Q$5< z_s|`nrB`s{8#}X^hCEJME2vh54l-&9zFeb=)|*)*$?a-(y}+1a*T0T43ymK?`Er`G zl>aC41l$6OvSZmyX(vHKG!d8@xe`1#Tg356!xyi0S}K0b`7_&8O$qVj@HL8$5X3U^P#4hPzEoz0liL?MOil3Q+EEj| zNJ0xA&hTHH|Inf!v?9MN#v1zG9(|a7PeQ6+ObygTAPWPeoH(4(88#K~57nYTW?mCsIe zeCmn`Z~=rlC*QJF1jIWYj64UU6fSj$m<(5QJDN<9|ETMwiq7yx`@x)f zqjP`4S_5Mk=i(g%4FlhI8~_o5&-Ul;J`lr+ZSt}(^a-%k*U7!}`DL%3^&$X3B__HB z_r{(zO)L_zWDt=>O9}Y8CuHxC&7s@&9&3VB37aJ3A;W2vh3`k>!<1e@o;(TkJxWCQWLHWcmSBnw2QwD$oMa4%|D8Y}7dv$9N z`6e!f++IduiS!3W!MfR~d=e~R5`WD)D(q$DWal#wiay`4pi^>8YvjcLFiH(Qhlj+j z2Sw6ID(>HJxwV|P2@J29fv8G<*IGDSlKUV+dRRUD%}}HHM=TDC3>x`PKCLvPWoTuu z5&;_kuJqw&xADQrdr4mN=J5Z0eQ)lB*l4`Rb!lUYKp=$0!?o^5LVxPMxp;?HQ}tK` z{gCom6&ZOJ);FrhrETQ#h4bXFxm-wmxUz+o8{Ed+pJl4!hMD*)%dO|ZCoH_LguST| z5rilYa!G}U$v!V#c5}kYtw$hOINcmiDRBoPP3W{!kiLOpQL)ad*IpTuQh$&s2l%Vr!j_mW zw^XS63U>G?N8g%c9umCa{f|MAU`SvB0>8par;@)X@(Yqstr$koKW8!G$D52Zfrbn? z=7I$tHb`zzDs1nm4i}$=LJiX)$@{X%Z%Rb=Ry&*!s=Ai zoeZAz!TD-C`F(Q@#H|BIA0jgJ)b}f89gEALcxeOO4%%xfUT@RvQr|oKrOV?My^G8> z&QY9yWICWP=b)!~HTUO1Qf{ZMw6c`$)KBf-%!Jgw*1FzJW}T0Z#jM5}US6{zX0DLB ziB#smvkNUBr=;#ZcX%t~8kp3DH1vlilsMO|b8?*peTU^_(j{%USdYoHxqu`rpH~~% zc)g-V$3v$>S`UT>!qda^*sZAmK5+yND`cgB4l3)9VwA4Tn0%$E!U-zOxYMbAc7Km2 z18SQKf~TQ}=ox>2>rr$p{QU8Qrvy`O=c8;w9#3(I4?wARY20PSs%b9E{BeHcu6hsQ zFMSGcpH$6MH+A0L3%=lJ`nNavR&AlNx{PXASlcN1mU@Bn0pZ){XAL_NVN`!90vtvq z$RnH&vtQ`hSotk~H}aa+jDizpgF|-%Rwr%Rknvp=o7*{OguqTY#fhtBL|STih1^7?f3ss`)%i zB?Zy8@TKy*iXN>iWR`FC`NjO3#PC5JY>ka-hxBG^6hKn{wHG%_P91oFF~MaC9d(HY z*Y~EKs*P!dL_59ML9d42c9Cn+vi>d|<1|W;?t7zPdF+Q9o^@F%w)ttG_wIPAK`s1M zXBkv1{Q^ENMudx`3=bYNbj;1A2iB6Jdwf%%edNW8IPfP4?hPm$`diwH6GJ=|D ztHq$4(|z`|IZT-42C+}f&(IXCAWa?Z+7J}?Bc9Zir4UOPH9b#*aMqM5#f)G!WPSh& zPP5hL8~&&h<@XhzF_C|oAlfd3z(a%Hg`)4h=OUMoSYL7EGsn3yL-p398Tdp9Q25#m zBV<2O4k!qCDHJzPbR7qIY-RH9ApXV=@$fwxGHaVhf9z4{C@q%Ls1oLV2Z-?<564v3 zU%Lt~#;5(zaa{U+t+n>(`&FCWONQ5lx?tRLJ`AO}53T{iZ{qqg{b&;m$%SW;DJP>p zA(4Cp3JDb?mcPjsOV1VeLD%?07m%x_?2(OSaC(c{?4xxVYFyl5+}Fs_S;1a_>{1{KDbqm*Ftecw?x=a1i4sTQVkSAvyzR@nXLN?VyV_ zakKE!3IxW72v`wJeo3ZUuoRjZB&)*eD^5| z;S5(Dvr>0%quF`Il=|cXU;J_|Q)om_<2#hJWfZoj@l^Eph(_$K8A~!W=vlDOP+n6j zw^~gQDHt4GDx3+;kZ!oAq@r(THJxRFTBV%$RbQuF$Mow_WkuUwzu52+3PycE1wuhE z?M{X;OAUzeAW|PJ+P8_(yBjrsy((SVNSz};4b};ln*6;fH#p90;2^Xuw|Von^ZC;a z@%59XXee!`bK7h4hfd^AVTNd4Qsa{G#U)su10kZA=Kcp~`wvv;;o14rz2RB(O6WSF z;(Zi6;a#3FPE1PBaQ}^DpL^ns8sB~cdt5f|QHssPZd@u)-@ZIo(iWt?9?#*eI)l?4Eb3RFca63mlldSmmvsvCUDl10?AXMP5+!x!4_mafDelP8939FH9%(K z$>?9zhl3Ef~^JdJ%X!4ryY%%T{7Nh&9@YV3b^F1QHP#WHIJCaZq;VFA+hOY?ZClB^G zqLmY4>*-LIpk})j;<^kYULR^A0%2-34hG@a1||-6ID~Y7#?kR(I{qloJCq4~XqlQz za&T6$MVI2M1AeW2WWM*Ab-Qy$U3!W4&?(~V%6PmDVYG9QOy@9WQk7@`1GFP z>@2IJnc??s7cZw)vDOE~6iDGUAKY&X9kq|NbyWKEBVQS2+2=o{k_(aff5M7Sty|d$ zsE7UC%5m4>yhL{^@kIaGn!|tx|H2Z&N0|b(w!3T5RgeBW{JHRcMwizv5zTF?o^eJ6 zWUil7bm_;$_mxPCH?FgrB`E@A)~{a5Le{RpDzlm>&ali8lPZLUS?c!=Mot?onsEs1 zm%015^?E5*jt2Xw$9V27xC5IDCelbB;Z~J@r7c&E`^YJ zg_JXvsH9E^ItuqO&D*YR&YfeD;ib%>tYXvi!|nb`Yl7fWC*GJF$>XF-nK8(NhkbA{ zULb`A*Lokx8KnG0{7s~zJDkXlQogSM!LOuxCr|v5ks!l;bf;ahWFNPa7?QeF%aC$t zwh&IyK<@K_I{!AmiH7)OP?elNXByQ+#!;1>0`fTNeoBC&p7sYic8y0-1jdXN)n7Jy z@d)u*^2-%b>ko?c?JwcF)-+yPKOjvPVIg8t94IP#4krp9ERJh#GdGL8WtgW<1VS9M zcR$CZ2__c+P&3Sw0NPOgP^vGfZxt17LC4WDp)iphT~%wrdq)$38@xsO7dOml&A!cq zU~%eHA9ne5Xc`oX*m1;}bS0z#JWKIS3cM_+M{IJ9;(1-e!sl%%m55=a1SPCZnEtp? z_5M*nf?<%jtST1CX(AK{H+>Fh6>GQdmago2u3k{`fU$E?kdsc3) zXt_=k+0f8XZ0z>4Uk%gUkE-|~h`6(6OPIcK>P#ofPm2tVQA|L=S{5HswJ4b;oF9y( z`SR2&614tRYzKKI4_Hw1_E`ofCVi0kiKtd|!c-08y$i2L7Bh7)+|4H>ozdvfg{Ug< zq9n7I5c`~lvV8q9RKPW-vZ+YDa1WC9dc-WUX)Q)V8O_&Dx8NMgiDX(zK>fS3#R6m^Iw%cfYpdJlo)r&_Sq7AVuybva)^x z1SV6m>pt9YH>gcMb@4X|f7)483tRKgP#SC_QxEm0{UN9C*Q?YGJMjufsHmZi_FNML zuYklB;(;wBOda~cM45bRks}rQ=}GZHwXhGH59br>Do_B3Xq@(^&lR4%(E>s>*Z(qG z@>sw3!7S4;KQ>Fjeqr4;ma?mTUrjW*&1WX5K-9RL4M#T ze3C|=o=@==f~t=RTci z?bh;!v0f)fIJ^2A9;)vCko&)XEi>)4FttygP`U-S6fbPOALO^eh?|g>OQel^@u*{A z`;*|DPe6%@iFJ4Id)_T}Mkgh8iS=wt#>*qTH?@0SUZgGpIYgzY81Cg_RQLPybeeue z!+V>{!vgT5n~cXk(xHxh!~~&HU5p9E58#z5sG*wmyQx42P9}d-U=GBunNf~ArKuMG zM3dZ1VJu7XsYkNPCD_(0DAcdMs%1t)6 zdGj_(X7&{lwIcgEkpfS0kx)_F;$&O1_Wh}3L0_1tmBV2@km{ol8lXw-yTg!eGgR9u zPacgbA>_q%f>{wL%-h}|JthG%vYG93_%vH~?Rj_rZiASBgh}J~K8oy9WTTFiz0Mr>DdwLxTXQds!t@Zq<^p|BknSL(brO@=an~s>O34acAWd7HZ1GT&L(r) zqxUdCEz|CGg?N8d)&raO0%n3vi{Fp`K1uBG%R0>!P>pOnnfO4^POK5^Sl+yq(%NL# zz%!V0aJ{>+?jzU!x*FzkA>6Y7Ap#|O$nCmRQ3gd zK*g+qbEKq_oNkBv0JlPx7A8nNJwiZN5MhWm&aSQsy9SahFUKh&+uDkdx2lqZ>NK|T zptbSLAEKY8IzP*Ul!sKFk(5&u;luIN9qUb(n<_0G@-dyI@m;Le5F{_ct+2VVEa{lq znZG5sH_3Cm#j2-`Z}hkUk~~43x3hm$hHHoNwBiw6IxRqByK{Ye2)DLej5$S%2x&G> zfDT?Lk=Mw}9u5Z@!a3XG+IzG_C(M3;__w-Fxaa`FtqMgd%6PhbN zJ^2%&e{^>Q{zvv{+p(cx#Z2ATPtt`SDWY&(tzHySc7*%AiD$XHb}~6OYM6QHzp=xy z#sUS_g?=Z|3y?a3xH49Ss{s~F8TJmT}Rsmf+EBQ zB3_Dxl+@WR<#2xJd%$0{GfF?y*w|QXdV1Mgc6K%G4+s%BIuLAJa*sIL!;~ zNT7U)oQP}X;UHpeO5LSr5>Cm;yPVJ|qEs!QpC7XD3>Fd)U~sLFlqOu$VzEOFhL%pt zt6FLSDyGCjsQp+RS$m6_Z^VGMD5b)OCRi<1Ep~P?zU27Fpk~VTq=xNf@LO^z?%g7b zHx1C~AZQf^Hm#pcLi#Mj!oOdIs>xWqIaP?JRKQxG9H(Wy3*lm6fRuP@HDKlMn=Vpj zL$?MSLfCwTV)n=R_pyK;#$TDL*{xxIAfut2y0RU!6M&n;hSCyBIiG8-x=to&PKCst zFfBxYhd>Oz{HtF&Hy1LARA}0&t88RI!8s=ldK%u;7f52|oR&sEZa*dZKuC>y`tEaU z&rc@AM}2BJqhs1Rhe=g2XAqEise;e7D)9*;wMSL=eX0D_{1%bxf9nx3V0QEC!bXT+ zAg}Z-Dpfg~zT1&7<`Zh45Ue+0KHbh*3O*ZnH^L;aY`T((vmj{sY%TDoVByn1QBhTe z|9yA1Ld|A6S_wghV$Fm~;D8B-@($$;2GRD}noF|9X9E7Vc?H>EL!}#-zf2Q3j*A{} z5RPc$zQ>&>CzF*^^$x!Qpg&Yq$pW0^PD_4nl zW%=T-Jr=Ic-^(KTLB~ItJb@DynDY0=YWR%mEK&mk z{%w5Y@Nrb3?~9Y2_Ry1cpP>9_wF1w2$a;ue2aa4Jfy+F+Jhua|tlo+Ty}N*m!nKD| zbABc`BzZ!%OVGpi7dzuce-g}y6ciSV)GwXpfq>~Q6qgkM>PoebR~AeNFTj}4Ew_gS zmtm<<`sZ!yBfwe7NfVjlwwo?C`Fh44v>Z>zj}Co?3qm}Ljl?nVMoRe53iC)>$P$R}jk@cVH% zanIWR=h~J|hy5oixx;MXDs5)S_SsJuhXpGeiCMIp?HxY42mn-7t{Y&nK%H$I#XxTa zKq2W9J2CDH@#GU}i|a)_h!m>qeK$}OA(U-CyM!XieAq1_USQ~$UjqA`KSJrG;`a*d z&hF;O3W1KhfXpI`?SnNU* zA?!|$`7&Xt)fEMumfQw>X_#ES3M4tdr!3q4oR^w!b0 zHl|E!|8>bnf4=IRet}@n(Gb13C%oz2@1g>5zv4+E6QH;+TJ`l?w^0~hSBW(JF8ZXf znF*+Uk=B-;f=m|KqXuUT!_sf^<0jV}hrw66z+Vbrgiun$w((+37d`Paa8^L~=iPBG z7*iB4F2__f7W03O30T;xc?>OynNEp>IxbWq59pBN-Y0#VnVKgplN&<_*&a)UiWitj zJHxIs)JoO%V<*3U$3=l)6hl^4kuRfq>p@uhJ)=e^-lP>d)J-YhJb&sU%kIq2gq z5MvaJ;t;uVF0(Ki`2_~ynR0T#8rpRU$@?hvdE}V!Dfm5=@0K zU*+Y|oeTdE0oAJ|vb-Sg-tJWE0=PL<8R)qVy2nsQ`LO;q{LcBktmi*L?5$nUcS%_p zaQ9QQI3Mpe9xJIw%4n!UbQ4=FrH#k-R~{9|4?UewT#9ujqw7#|a&jWbXqMdXeq8c5 zK3O&)1Gij~Rz0wXY9m!Nb;Q5y&!BQsD~cRDwkNXTB`yGdU-x+bM~|1h7ZX8eA?hB_ihLL50#)~DagZQPuBa;9pnPO)z7#B2`Dh@Nl>DZ z`X;cLk8+m=;EiT!Ag=5XS>q()K{PUxRqxVn8~8v;E;4RVI)NMm2BPYV_F~x05=8qH zYCZ|`cC`%%UlRf&)8dt?MRvjH_-AcB${Ci!=&%@n$$3A{qHHvGc(8>z9O{)Ie9irtkblDheKx&o!3W z*lv&d#wT~DoaXig+qJCaUyE)wiCFq|D=3TCH4dM!A<7GUA$JDX%(*o4SLqAv0`8e;=^~;ObQipFx z9A$=C3=BOb^hM(XK`_PCg0#=Ed!A2ymaT~UlxnYw>9P|7O@3OPOY*6woqEN+2h}3+ zinC%dKs0w$Caza=$g<4>zG~7MS-BI4Vb5`^Am}=>W0?y>H`qpE6l2ZJK?m7&$hZdE zKEkFsi3ismnnKpyTFxCs&pV_p#F&bOn?4_UJ zr{&uqgJk_&zrd3SOWuN~#tbJ#Ak3E5Wxp1~W#k(%Ifj@sw4~RSQ*va|szF_hhdk2! zhS4I31gSb`Ky=x}(6zQshQ&P3%Lb`BR)WRm3k+lni)E+cOH=|fi{v<`s5qhn6VH1^ z)WyBA7$@C$_2F5Aj+^gsSIzKNy@8{1lb^ zp>eJT8E?7tnMlnedw5OaT>6!p7Yw>*a+D6E^-H!k$em^=)29n}*MeNX4#&&oJdgSn z+6fMStd{llpTkx}1df@>5ZxWe)pVK&x_I<7JBxb09f6Kly>CX0_o`Ea5Tms18WdKU zt0WnLZE)FXd~Q066>Q|@!WmQ2DyAl7dWNuSJXUt;1Hn~LEB8`1;4ahpGq$b-FTNM7 zPZDOYGf~CCLMtL%?{6;7+|IPrx9qEQ*?gduAC=Aa*sG0o@IVV`>=Omz-4Q${JJKO!mZtyqTX-$tK9rka{haDD&BSme#u{Bt50Pd zUfCdLXID|uM|*%nI-QdGG#2V$4Oa_vI}fZ>dGw!IB*}CGKVNk%unOeA6GEI zEq6<~r&A_4)|h3SI$dJJ56+bfbBO5>7hIz5D#s4j$#gy=P_v4cS#Oauzw)aDYBP+yP6kK9=OAAaB4kK(pJusFkvM-&yewfJh>enrTu zM=ED4p@2A%yUX;K4xOTNlOI=7)1UlkV__Z3km=nIrk1!>RG1Yj7&uwv^S7AbuqYRV zpZn+%hUu}6e@2k3NZ5JNXzTlkPiY-F6Be{UoTH*n_ZOptmVf(_nT$LY0?VK_q@r7< zvMD1SwzX| zz}w&!7J~8fMHNqAe=rE+Icg+@;Y9a*_|3-C8UR;u3l>Sd{26XC>h;m2Z%FFp4*zAV zs>02wZumGEXY>}0NI!6+fv-yz+0uctFEguyX|!%_o1V_+0N548g=!s&A!faGB%TAH2+ zkT93MoiD{&nCH?_qx{;gqLtf#F2VRcs6wzbop~dm_afWuT5^ z*={I)m@1`Na~J+JN0mchXXscoM7Lw)!FK&dYe8Yb!{g#WuUys2LL~a7rbbB!DrrV@Ce<=Qcts2;9SlTU+a5Xb`pUsZFQNOgP zV&xiJ!VUriYN~u#L?SH)H4kI;vj9JzNWW35gUM>aY|_VMzXLUei9%U7=Kqvb%!K?| zMdj!iu(0XKP_;N{gk}yTx0}dh-Im}M z+m#l#^QxH#-EEcZqgP^?k2FKLC|Rl`MeIhGUKP#8d}0JeLqV}~q_kYl83+7~%^K!b zn7HU68b_Eb_w8OJuXCD{b2CYP!?|CMuU0EGZ>`#z1=?0O!QQRaX$;D4|LN=$bP;OD zIfx@x;>UbK>9pTwS3fxyhW;unbaPjx9m;hzDDjMmjJ9s98ql(7VjP{wt^;I|^zIF} zfn>*ndq+*G=wUn{5R#XvY7*9_=lB1pddJ{MyZ!q+*2K0o;lws4wryu(I}>wa+g8W6 zZO+8D_3ydwbI$X7>Z;1CRCQPPb?x4Jeb;9#)$L|gF-ZjGQ%#h5i+8vpfj=6>!(XyV z$H+GAT(=-5m8N@sPZ6PKsRuZ@Mb}1k={>LD@_-Dq!5vp>JolV|;tFx7xp9IKdlF=G zd{LuU0=1p0fOg-nD*8h*fl=DaUrHgOHwSdAmNIZrPYB+}vj|oFm2U|*Qp=2>{WV;} zUdL&mz)0{RuqVtH!y8VLSR~}umrO8!l7Sd$haojM@HaSX1o14H#^mtuaG&>}eh{jG z46J4E3|O6AhSqdl{_2}p>*~(I`t9ZhAz0}%n*%YZktSSGRq6m2s1bnsq4xLRiHI^; zK(5XpvFOj4S!2wE1xEg=ng3kYNTpQFyzGi)7Aa^g0>)dm|22M48 zL`bxqEKhI%eTM&D?%0Gg+(e_JGc!koyM#l&Eo+p=5wT>?j1b`gG-m#X;J6j!>|M2D zs^Qh^nY1zC8?gE3VZ=a#X6=Aez7$9=xI%sHehaEgQSroBzVwj`YF-d@m=a z?DD z$QXHdBgHSH1dIY5Kp4xUv|s$%U)knXEE}v+jy&G8Vg{V@ySs3Y?n4M*`T~0Ge)NoA z>OzG0*g^=nCvb!7f~dJ(6|auho?|9r0W*<89dRTD1}C|~P+-znSLK)ZZ&S&^@wp&n zp_GmkX3DX!1LaoT7%U<+JX_&eIavgoOvR3Z`h}65b1EzvSB53o)HrI#`tS+>g9p>R z`*oHyV3UP8j;7qo`a#4t!`ppXKmQn7>2nV2qBTW@J$=ng z;6;)&6!VEP$z)VBh9&)4W>NT@o?xn9XJk7*ExNE9hoQKUHQ58Yh5|wIG$0f`Xd0>A6Lx{8r7vXHGM?+YC%Z(w456udX_j z{Z_iPfg{b|)a1{Y_z2hTm8EC#Kfmo5xUD2|S!qqm176mjP|-w{unJFOdV=%0nKha$ z_oaesU$3envFd=EZ`dk{4$6G<8jhXXZkLj)P29(gAe8h31WMZ>DWWq(L&2@ST{|yp zc0hg)EfGUZHOVtM4Hu~3{d>WGMG7V!6;$`(mCTC$;0_P>E96Fkd}mG|Z)A$;Ctr5& zK*|=Hv*s%Qz=%f4#>m9E7sftK>vYd9r!!0vKw~y9ETLX=t`Fba^=U?_qo6OIbVo5p zK8`&2>WTs&f-$D+ud^?4s0vtlB0fq;RCdZ69N#n5!h(EaPnsFifTaDgfwN0fcz$4S z#rEPtV&vDp{VX|2sSF8zSIO^SwvfODFOFpUMa?7t(#+!6_eRrWy1gB};U}9~S!}qf zgoG<$BgLN^e0gxEsrQv%F5{-Jr$nXvII%{3N2ug6?U7X|KU8wj%j+^NkF+dD#v;di zw&p>7)Vp6-Wt%R5i|56=?f=-x>GoBw|j02+G7Q9VD z$av}{@6LCaOl>%L!>gBwHR@-6&85T9w_+JZVwm-Kf!WRIbXyqFtAOzYhuTcg2ucJ7 z1^uB}$?3vqblJ-6E0cf`&)MSRj5{|uyi;6T{W4(o)#X=E1hb{MYG6AWX{2~YyA<|j zh_aGcJmBOat9@ye|5+wCe7Tw`!|3a|vrCRaay$SQ^PGrpnmfxNPwzM1Po&Z*D7#bc zgNA>WxE{#|K@san8vze|K~Z`z?5%E@El_ud53909?Q)ygef_wl)8#>;%>Au(Dy2mj z`pN^db(*)gNFZ^&Ado9B_^|2awl6A#^j<=7rs~ks#V2x+flmaXO7!GXf6I+B*a`$A%*@$_5kY0MR+j%IPe-izfpC z#(~gy;4k7X=>K-fjV6w#iu>N|!bYJWiR)Rltfi+!>C){k%~beJZR|AESb@ry(JteA zdm$OsK0(5K)}T@;?f5%s%i!rn0S938ZU2jc3u?KM^xGsQiQR50`k56`)vd%Fmm%Lp z@58Foom1_3UFf^vVkkzZS#cw61P+pJtL}tjO&20^#}$nN9Eab2esxznC>~Vv;qO1u zLIA#AWH<+unObnCw#Sg$slb{mcDuq#xLI&A`;7%6#rzfE;}kBwz`iKWM7vc81UOIl z(lB~bgYh~!%sPK-Pphk+a+_hbSPee|O{0gdRBbFxm@_1Fb3E1Gt&_lL)$u23f(LDI z?4yGcsE~t^yw^5V_m9{!Z%791Z%imu?6NK+r`3M+>-wQl-e(%iNoUqEH^6C7N*hy# z^EAAqKEj&^0ov~;ng5+$#UBJ@xT|ridx}k^Bkx>%kC@!mWdt2li01CE7b{Q7~TUZ=fshEW<`+MC#iW^D6EY*jGcC&Lg^V5JSjiXQk;hIo3st2dmpckCrsr9T3p0QipDk9+}s z&F3fFH-NHgT=r;M)Z+DY8dI9fNBvKJG;=pVbU^yBgCoJ6y=&KfUVce_G2&+?&hR{i zBs|s64#p_%P6&#a-_2xfyuDwlR@ZYro_Q*L9LdVHTxkTOmbZnDxU_9a{3ITsO zxu_~XaBXrz31EcxmeO@6BW-9DB;^S7MTs6_g<@J!VZbShDS{5DKqz9BDJ(?7gc-hu z>BYGzUdwP_P|Z#V(`7{Y^Uz9j{+LUx>i#i_y86E4I;<)_@ED%!dIM1E(GImp+**I| z)l;!{Ur%YuYR*MiCK*g(HxF>3HC+q5 zrhd*%M;ykn_^pkIf1xR`ZD85#zNWqFtLN#chUI__gccWwk{d1X6a$i1am1tZ zGU!uk(diYi3V4lK!O65d3z;XAz=H#<92F2zy$2k;({O&mA!E(o!ZUsA18SYsC139~ zV98P6aw!`MV((v~${dJFi%uDeMF78s`J2QLy<61N2~a0I@QwLq3XjiQ(_@02`WEYu zLZ15(Aq>lKEDK=JuZE@)5|7w}LbNPKwyp*5=t1=uZbe4%T2a`j3Vgh_eNF|FKk9Tt z7l=8%uDw;EM&Vr%eJ6^kR579X5MyG{H9Ka%%@c!}>Hv{sKvsGZmxP?~otAmZI%X=8 z&jW~}!Kp|92Z07rVu$6WZAO>LySB7R)lTqMGR8n{GWp4VcJUM;ROKSBFDCMkvi=ec zuDLCj|FzbVc;dG9?Lu?&J13ywW!XVfzq@2-jW3oC_N6h|+11@3HH)oTeTbyOv?XfD z3-{;Y|9G~RN07@D-(4z66|a@GEO8Wf1qWA6^tFf4k{469YGa zuDfB=m5*zd#d*twrKJ@~Iev|@a(g`T@K(?mgr7i2B2tGTj$k-eaCulBz@SY(U88w< z?&xPk?w^@ zwj2;I?I+~i%w*IGYhT=Xk&gIPctEHY!bY~7Bx$6_6w_v^e|f+LKFwk48=Q@_*;4Lr z)*q!lJ9>%aG4dD0$3*zas2S1)rXxqCDa;4h5WX3I+oZ`oKVH#LZFD__T8yg0j__bv z61CN*6$vZxZOLR+dOMBe`%FrCnbQeC&v~6n>C}_R*}X%l$as?2R-_G0!jajc%oyTi z>;-@6W?VmvssEhV|ALV^u!8Q#asj?u@m7VsY>Xe;UB@xBeKx9|t*WY9Xl_o*^!l6a z(jEW(S(fmDONfZ~GYPBHVX)Sk1*rCIRa%W#kbB6L%bZ}=l3?lfRhpC#!{|F(kI`U^ z0IOZyXZhauu|O5@6(mV7{K0SgEAXsC!Gub&mB=#LZ3N z^F^bsDusiE9--^ZXYC{&i2Q0oPa%eF`2IgQ3wSRIMY7>InToC9Loj!E9UI0%;6LnV z#dmtN0=XI!zuoej2owp3B(0_WY0ZH^<9irJzbjt#7QoU5(dF;lk{#&5TAm zvBYpqqK?v;>hiUdKiEGIiSW6oz2tKeyI=k+Q+%a#})~iK=hg30SGc$@%7hb9sLC~`%c>$URp!-KJ%vfE&s@4fLsLoRt%CbpUqUZfC z&xXs5PJ$bGz$MwcbbaeGkyYaS^#*Xa7Xfb zrD3zO!U&@aF!6~gN^+@5{xHY`S!96OqVsWX0wc!ejgE*+E>T2u!vXK@wQYxPtPe!m z6n%nqjyteAJ zwrakW-qhBQ6OAc^v`NYD(MHpK53pB154*5Fp@;kFVa-9;w==c$TLiiWJo{946+6pL z8mr1j!?L^b7JB^)fBMr!WFsj`BN4FX26qO@C3HFvP4M*$M_XFczb@gWPIp3tuG?DI;C@q(R)eP zkQDMYANeJes==06QbRDeGMh!-sIO0Drl|t!(T52w33=*gg?b1DOy5b&p;}FIYplP# zlUD|7Q;$eB#%{?IePvTT|pjvH)IoO8%G+Me+=|n9nFn5+Nj%arzp>r)@*d>@m(JI z;;5HbRJy{!oU)r){X?36Tj2XfClq(^!cbR63{OWVT_^^!pN)_`wA=nQbv|LR-xQu# zk29_0=@S?{l7Cc`nl~BcO7H9f1uCUq(6a%^GOQ(=#1KM2j9C+p?Hy-iLCM69Z35j0 zWEweI9>#@vVFVLUvqh(>NY@k@0iPNe(LL|N2(FEAI(;Zl1OQI)%67dCrFe?K&_TZ; z)Vj~tlmNVOil!op@7rQ3dEDNjmu$7(4YCBV1>~Vqj*|LlmA()u(PeeiH|N<87#Ha& z5Ra9kDWRRWm_zS6r1~)bqdZ~D5MoCjwp{1d7%607pnEwwZCq@v%o>}lRyu2@W|A5! zcyO)>2Qfj7Yrsl6-1TEkeYbT=SnrzKEgrvVbcXmLQINxsL>;0uL_*zHj!xP0X7))~ zCjK*wZRAhoyuGcj)SF-m6C^H9|132-|5RVqlyRZIBYC;0*&x-(&`;^Tqi5dd*X=i* z=kjyU&<;2EZg10Zmi}3SgVZ*@A5aGUg1BWh8oGt_D*}RDw|(Y`OQG3^oCxWE7=hI8 z0#Su9%*=mRm>sY3B%y%DWp73&))SYGXQFPE{QW{sBv_;*hY0S53a})mqE=}ZQ*P%k z+{E?xm>7A~o-0VZY*Fuf`1~X(d7pWfKk=J;FTJ6TzeE=|@WZslO_|@)POr{##e>lb z9A@SxxwU1dS^j%7$nbu%$=SFOq-6nLBm!+2ou_6^foYeGcd!p&YsT^ z2k$xk_Kj46WEvpJQv?7ZvQSyqgROPddI-6=g(Mcj$ix1`K^KQ7kBCk8LObncT~yyh zs)Z!Jz#%A+Ze-fqo~~3gZhLfDI#mMZZnI>o@(-tyow9QT%4M#4BnuR@E!}x(`Gb$t8fcnyz`JI;9D@*l?y|mG{PT{lYI%FcqQ{exZiaf_?c4%9iAa$7 zr(~pDS7TPU+a2Y;3yHOkv+K;X9u-eWXU-Op{3+bmJ)5`lXT}<@_MB*Yn;RO&fcaCH zfx=>K=^4V?U{%$DuoEV(RmsSAod^dPjZGC*3(n`{u_K7AwDs=%`pO<#EGwV*d7U;L zbWwfiu>|B;p(S>v#8_DVMvWMRdFEnVniY1>iEH7*aa$*Y@#QE}E;4_pKxyqdD_~I- zQ%Ha7LCz|ax*`GjZu?figy~XMj_6P#UH)relZUBUg6Y9MtVG$quzScSjhkLLSpcA> zT6N^BvFJ$AEeXsRX@@j)_WCD61NaLxMu-g{mh?eDz89e~*-yR)^oDBTeOYFJjPB*T zBG6L`O4^N;Q20&5To1IWm{Jaenomh50E?e_;D{7$I%^g&_d?03=abNOTJ}Hus1Q(~ z^=v0lzX_?FNB?Fr_3r>RPzT~j&Cmn&n??#HmKp4w9Oe(synwCK#J=J)!UA!} z_b=QC@W&p+v|?Ww2lIHtTepRaM2f%#${Wt>?bRe_;2bt7Nh606RE3N~Np(beBN)+X zK!RV>TS>=t+C_d#u>A8~ejDJuf-M)i7?TZ^G8hiRhJ>S~H@Ik#@uSLh{9;^H(q%_h z>R)VkhSs{d8OuTs#z7miA|^f0Y(osgCX6d2iw@_>WFMvZ>QdTPHn@#V{zVo6en>y= zoS-*@|7AYpP+3H}VU6LDg!P+3Uy!g)pz-_a(0irR?h<{Ot&PH+G07E$&!37NUo`6d za=dETW)u|sz+J?Lwe9FCddIH9Ctt+L^S1RE!e2B;_cPeoZx0PQ7ib`V08SiVJovA8 zJ3^$GUrb6B+H`@owdabmPSw62F`^Kmh!IYym*<7R>A*Shqb+m!exPC%T9i|osv8i~ zyBG$qnyi{znk(;tuSbhJiRe2lq@}`0E;!y(IT`!4MuYQbrjW_(XE(bNukvNQCn+!$ zDG{WtJrvTapR$nhtLv|pJ&5*H^yb4PpjKNb%#370H<~Cp({<}^aaKTBN$P>@U@tCQ z`)}kSG5ED`x%|E`oWVqxYo!NU3GDVKyXUPaq}tJTaTqRdSE#SYgQ00 z7j-lOBa%W6rAd|-a^aIve7?$V{4IViAiAKAn;cuk!5d^ZXk`?9wf^H+HfHkW4%nQC zo?``0Jj}e6e+$_nZq|MtLVWPQ}=pV*G#-@*Z!!p(F7<_ z`=3jbfn5Er6S+}*P$cn5S(MCc#Bcj1U!_5RjiR_Dc5*A<+#TwIR52YnZ#GWk^OB@z zt26m@Kg_e2738(}XJSv~c|N7vVmYx#`(g05p5rhWT{T%++N*uju}dT_B#N89;=pVT z%+xiytvhp~n|S+ma(bcp(&v5s7dAlwNS`iM%&0xmSv(X7j@s_)4BgCt56Ycg_{{nb zv%?znaaa7;>&Qc!OSO%|!;iQ2&h%k1(J=wvQzXyHN7jO(0$siB_W7l=Ix%Squwh>0 zV(@5np@PS??5r!km@m^?Yj2b?iN}hwzu>xg+pV(l1Lnu^OYZt3{q9Qut2G~k+a6-W zF31?>pDzy;R-IzGudf%8ISu!D0?2#r`}UtYK0c=NW~*)9NiQ;EwabqezeuT)>=pA2 zZ!}fP=rj1#-F6eo?(r$#SGIlc-OxGB&oh_#r_oLzx%BkgHt$9(G}HN^{cxCVty;2e zfs((Tm;dO`eM#&>{)&jIosSW>nbZOI)|teTM&3-9p*@vA;^lHHMO{VmGIuIUklx2F zqd=k=|EdlpJ1=v$CFlx)md(bWIH(z{1a-7CKU{_zR6rxcJu=pa* z{+@q7PQgv0#*ob&>0fnQj*JMuuwftu+Z$KN#e(tF>nLglG9j__zzQ!n2=k=J>w~x; z9Nh^eG6zwVn@$dy2J(A`f`mS$Mi#+jX%-Rh3E~q6PmhoqWJf$YjB5tqU{=|~S@)T0 zJ9}0MpR$N8y+ogK4#a;1f2-zL zJ0_ztz=S%|Ne&R0*j^s0T|)8 zryw^SC|7KFKkH9B3deT=fk)Gn0FPl`KA&z4EmFVN@z3Ok)8#C7({CAlYoQ<$Z&NHe zzt|A}tW_K^#JPcP%+%)z^n=`!TlBdz2a24fyw9smjyG!m40sZFy++cxz`|=lQj>g} z$Lmhc0%7?$pxq8Ej=nuaR1-V4OAKxE77uMe@7ydA@6t zJ&|XepYQ&>pAPPhEI2;bINWaCp#A%D&e{Y<_d(6}o+g4br}0n|NyqH!))g2Ex1IW*DxhZ7Yqz|4 za?>4!yEmWS9~Jx_{QDlCP=Z)%iQj#14s%|m7(vKuS`w-O2HKuEw=`VVw96I-q*r8P0;Z7N-j7{H9{+MO_c+ zZCgeljyJtdwRoU&6)Dn57qH`Tv1E9|Nwel}womJn6EZ59jn!-E&;PaP41xELf+sCo z*-jsWCQ#98eN`WRJm}iIsn8`_!(_~6)5a_=hOL0c)zTtwE)@ihfPh_WYIJX_vA)Eg zH(uMS=wg7Rm+Jt~wQeE*kKc1#8r|l`pl?hg8vrw-39arnhrRCtiM^&)BL2m4W%P zqH{a?-oGpQP8cg3wm-J@-v*%kC#18#Rw&jDND)@|TW2^MQ>--aVojAxl~5(qHM<%D z7xueR@Q60J%Y=pJ+&bZ)JC({kAiE%O1@g#a<>gs$`Te9qWgzu{WHwoFB(+ZW(fSmT zMd)#_s~~r-?MOvFiWngP*>*#Js1rYWxM_FMtqAJ!@d8^Gf0<=F8oCjhQ_d33H`^Sz z%?OEr_PmG_F^zp~$ss{jbp6g0^d$v@0D`qNtH>CEuH@IkZW(^%hGOQWb z`=w}9%%50rKv}w!og}x<-)t(+hWG+UDm4pC>!r9yr~(cb22Sw$cXXhTqV>;kI6XJc zY5voK*ANvaH5!!^7rr>JhTZtm2^_c~=rl-o?1+dA%2$Dvsva0H)aodZ;kFWXk3C&8 zVxM2z3HJePF^n5o4~ zL&n~Hmv|-tfC~WQ|IwD;K`PjAQ5Fr?g9AqlLqw>gU^)bcDL>Q$8k>W`Q3>yP_dE$= zJSh@N)t$Eg-)95}15gR>7SeN3%K4qKQ9v$Lw735z2LyNbVD{B<8Xe5&h(b^pQ zNT2UHgW|-VG(YZ2M!jFIZnV) zUI||Xu0C;NzXRvRHxPmA*>@O{uVNX|yRQXU`p2@2P*VBBC{Z8VAYpLhHRU^-u>wz3 zZ=1`$>QyD5n)B~GbrIq~_vUo;my{#QN49+@6@Cyr+u2-aT*=qx7$lWPzo)Vk)BTNS z*NibqnPdr}!6u?^Mp}D|lrgEF#0b*~p{6N{T)hG({cqRi{EXk{jl9ms(4^I2rnTSO z6Z#^SHT(j~`pup8x3@h+Y(vvoKfhoQPXWeVJJc-YYc$AmhM#SkTCTF)+H356)IdGZ zfipg2?Et5(KKE{@o8Z1#8qz1e*@ zi*}kJQJC9O2<}&&xYg+~WH27K+-Ap#5jOxKH#B*36rbKYAm`Y%@TJ zTUolc1#AS`)&&}Dj@)xG+HgYM&XO?XD;p?qZaTf@zi68*XRH^(kNRyL>2MP=?J6cH zw8n=&8qMi*fitECOJj!F_!F6k5mWDuNoEZSWciP~E=2FQlf9M+GBa9kS;w(TxkCSE z@<}!ez9l9VT5DcPWxy7Qg(ZIo8xXdF19K#I-5|2AIeER1!cV{F#|xB3Sp$s1gF;w# z{T;#f!z%eTfYDW=T4JRrEoTr?L!q(LK8NEzCk!4BGQaipC-6vaeu6ANnKvnbkmBWd=>%W^Pw}a)U?6x@ z-pCp7hK^m0cZ<{(*{$Ar`NLNBw(5845F%k0^8VG(;5TTBOmit+nka(;j!Eo`9A*5P zF8i?plV#Qi(hEcZph^qZIwcyJyVtMKQ0{N^o$sB^-Vc_suLmps z!>1QIq@9}V7euTdP}Tsu+Ia@Yv)FCT^MMEy-8yB6od^uJ|8qJHDS(z#6*IggSe39z zBtVa&l-!w+{3<%*k9kH!JYA!AS(#=__%%N{mqt`w5uYz--5%LQfI6_z2SfAd!sczo zLCX8(FDA{#0|rM!3zL6bDJeT>5(Zw1>FzrTm7=TElMd>eUOex(y3!8@)q+N>BZnL; z9*tx#7&lpp_@k1+VM9ephR$LlP<0!a7aGuJkt;e%&^&o*OF-b-@D)rTZphdPI^@{4 z286Rn2>IpJaJ?_hk^vOC(;fl^Ya)|kK_gFdG7Np!A`Q;!-WW6Xe-kI6F5~7xX=*C?%q(@+g8e z0Bqc!XpQF|;ryEt-?uX%E%=4@)PJtw=6=!5o@%H{_c+gkd%EJZ=^BoKn=kpFAm%Bnpb?TH}y^NA4L7 zaOaKo^*d9}E%`wvRr9*bXU-2oJLi3^MMbYc>uv`7^pC9kuQ?Mtf^fy@r4V&u>2>Nl zdJ$!z`DlQpi9IF-Pw312O)=G7rJk}`vGq8}AbOb#FU~z;qhd?DfSQhKUO8Kf9rYc1 zv4p3bXT4oY5(CFo471?|=la32soCls7tCA>xDabtf6}rkL;d~iHj;9XXQ1P9I$r0A zSyto;+Hg{TkccM;CI~$eDnuS~>mq3_Be>;g?k`e`4623;2`BB0k^}IrjrHxhwrw^9 z-Q2YHtX;Os=w*1wDb2xw=WWjeNFH1K%_;6IBn0Ft&0E~kHKRObVOc!j0s(#=pe^ak zS8q&YF*E$t3<4qe%E$YBK9&aMcHEqi1SC8K$H|QiH3>g zk0%8*DzZ&!D;b)A3^a>M0+;wvoS)td_YKwP;G7_wlq!14cet;?aZOV+2-Y~H(B*U{n!`op{z2CrxKSE7!e1n`l`hH6 z3s;fd5VfRoaY4r-K?F-dnH7e1clgPtzc;O$!)aLag#@etj+cRw^3DgZLTsqEto2|N zfjtOvr&yqX_lzk3C7Sf^e^0oMm%&}>Y(2A2OmKT1+MhLw^NFC|H&@Za)R4@L@1Z_7a9HN`*RC`e zNoz}WcF%(%2&Up0SgUj|At*Z#gJ z?`Kk>_wr{~ek=s?Vet~UiK0QtHiu}?fW3MBd_cNEZxkY&)FLGTAp%+);8UT$$ti;e z3&Lo^{82`=1>5V0Z73(h#ce=fb1B!gM#I6Yu1G}#MH@;QK}AL-EJxKKKyzV2{_AAb zfl_rOp)MLd`wev7y&B+jOM%gf*(iA<@xUZw5XY-`Ga{zy%MAa!DHfRk?QidUA}HWv4cD9kSp68b(_G$KUH%&ryKTQf2i_vgz`W)xf~(KRxo>-;Dz zGej+X131NiUYaPYFf}w}9LZf`Znssiu)5H7_WqR8vrP%!Wa%}vZ$y8CPY5HRE7xcj zK#wEijVi$#OM@sTBV+Ym}5Gv5DTNnE`W!e?u^y2rkOmkQC4(#jin`f3^?XNS^S#IBvLtQtO zFn)LJ&t&pg_CciWA!i=M$H7HT0~TPPHjFbUpftC$D{=h&0UVE)$w)DT3;Mcp_QpDk z;yytisV!TY#~7-})B_DS#$0=;dMtC>Zz8121w-Rqy@j=xQ+flP^;rK!=`?GUG29FECF{ZBZH z+0!fCYZH$1`fT(6`JW}RaWCX70?qsvQ-x%S2dvrx3vh}eEAvxOV?Vxk{wy5-D|B*LR zZDDy75BhO5>d3;7Ev#7_WN$Z9d5fJ$={#GjOACnRE7$mZ^xEsI2mbb!`;2wl{?+|{ za(QR9%h?Yn&f4g#D#HzMc+A?t@@wvbD#}1Xo*z01`B359aQG$Igqf&;3&EA>Cf)^8 z=ORJkK#Q^T0vUN{cb7gFEAfNSKJ>`rW0AHqd3!pF*gr_57pA37OZ;`svkQLUL;2IB z6h}zH-oNn!e_2!o`+Yp422P06mMYKOsi+MV-Er14dHw6)+Gn@DuY}%ugYqU(>Cb+_ zTcGH^ps(XzXrtMMilK;{e$nrI|?$y3(nrsZIz25vEbBs|9ZxiMdgKM?L|)=fE_SxTT0S z;^lQ769K})6wsADUiGD=4H}POr$WwdNj5|A*^Q!GT{e!}fBhBW9ndiN_~YHwSmRQv zsv*sYhW?Eh3k)3eDqJaJ$CDag4niiZx?lB%qZdP7Z%OVUVNrM zx~hqUd^$fAGL>F;sRJJhyYkeagWjzV=zn-oFO|m(BJ-LbJO$ORyLpv=r~WvvH0q!C zD;fE( z3p-jq6f;X!>(7r!W&>}|KB4Wq8SF1M>HR*Q8NF164s7tuvakJVk&vmox0Xn-y)0=sv)p+Xwr|&4+P4QWKeWSd~;jL|X{{a@F z1d({(s2V_BJ@G_?@Dvg9Q9Qw*3!7*dyH0d!@q^Y7RGbnql4u0UF;q#$iL?tipYE$s zEk>Z)e62p2R1Zc1W|9=Kxj`|=rHOR^rqkN8YPWS}sJfuBnLdz|wW$fT)=9UlG0)U3 z+{^&@l%Sbh|2asM0xftnX7HZE=(STHLOEeoMcJxCKzBL#86yR~-CqW-0q zRJ>InF{_f1rcm6wxm@}4n>j*h!&BGJ#K&%Dvkk}ygvsvT*jo}|zG>6qj&7_(ttkKt z`hh73b@?cNx$C`(P}f{K4&3YJwh#H@ssXcFy7%GpwB*5r%JWqBd%*(hf;6x<;C!W!0x`}sG5xEW>MyA(GscQDQC;vOL1wO-D zgh#C`k9zEM2OB>ve$2LljcP3`7fX`dHH?&YEgc^eVF$rH<4l|9C?*jCNwj2l75#-| z_3=7g=ytGQ$pNFYnh@F=EdQo)sA=1 zQ7fNnkha83=jgc{u6ggc?aCt%>@EfEVzw$P!CjJB3Le0GB#nvBWA$G$6%5w0>P}2r zaGS_nU4w`zR555^h*0KkvpF|n+RJ}`*1Hk?P*(Bi`b4@SmK1kPga;!fmQ+F^iy$3h zV&)Ffbit=OpO|_xzx0{xd~9ksZDDC>sE*+(VIqMG4D{clCZ<*#<>Hq_o$>QQ$_BI@ zzd4s)d}JCnnmi>lp!R#4PM6*0A7|0bdBkzDpk73WrqGHZ z$QY2XtomUWn|CtDagQ6{3j}yPrhh^q64)GOxH!#N5kynY?j4+9H@gmE>5WxY>G|*H zBG{BNm||GY)u%o3mIqm$6q!GoL(G0O`k4woKW!pSGSuAS|J$bACis-_d~}L_R|Cgh zsZ&33hdr}y(UK=s#oZJgh>O;#w=!B7OY=y{_jz>g@SThLkN#ugOR@5Fb#XHX-CPd; zxayCU*PNZ?`-};EM)y2Hag*84y*G-rf#HZd>yn?zz-@v2sx8OUYJ%tB6VF3pCsx%djXKQ0E6l5E^ANK9SK0z!N^g&7 zm{ijN7_nBf&u#L-!+1!>utc{hFhXa7dcO6(?BP+=Wl*d7&cUzsQ~lsV;9cJ<2IELc zL7r@!8fH&QPzW{}^%~2?N4M^Xh0(xaA(PA9pTnrI2qC$}g_WjIMON>k6udgGZoRy4 z#P)|{LvWPH&hH`j0%Z}HVmBC-7W1uHoANC7cj6|BRvG6RfYcxMky{xc%V(khF~#_n^i|9$Og9Spl}t>xOfXyx`i;S*owG+FtMvc?;a#|L%Ym z8tZ7xskVqA>_mqc5uG(zRnPM3@-XeYMzn*Ynbc`pfj?RzLt8YjLVWL}l}*c-@TtH8 zkw%A5r0I#?>R=LLU{6{+QGLL-5DhzkejblsS`DAu5qXsY)e(&|!W&I&xd>5hXYO`v_nsR=4 zi}#sTh^eD1675ai=jDZ7lb+hWyI+o-yK7xu z9Ply@e;P1$%I@_~*murSzYr~WGoDWda}1?(p_ND=`MdL!DRBT9@wxBr@P~7k7^<-Rg)jyv;E1CrMTzNWH-0aXzklU_KPyucP&GL z97~MvZuOUX$C+2tmm_y?`IQ7@JAP$#E4(_v(?WG=c}1Vzy3kqVcDN~rNoC| zFZOE(O?Z=ck5p+$`b5USURLvmvyH4f7{q9(b$~-8hr~H6%k|{$bTW+p-WzMZ2m~^< zLclMWKx&vgXO^5vZS-VzsopQL8e487J*Acvs9DOJSB)1dOLA(y7yDRid68r9M~t{o zWzQ0cp4Q16X>D@`gLwHXWy6++x1puYJ$Cj+kJE>KCyPu|GVV??O7~CeJ?0Sq=L1&F zb?AH%N2=EKn9aZ&xDRvdZhiEU(6LVxmp1LX8eSw_+Q*H{=*0ZPwR}ik3US{j% zx^6JYVl0u7qwjYMJ9)8E-qZ>_@oxo>T&|fq3Fuq-)<0}d5;JP##*@YD4qj(Z$$j!Q1O@I2nTmpQH{>&9lgLk|G?!i#j+!pK0#a|bt zEh7%xZIyTS&9_|bAfzjOa<`dkI|6g)4JHNZ(V37jML6s`(^uV z=(tQFXE2n;aLD&9ZSqVG*{)A$FxKWnPo(qwhQ`vtPcI>9EWr1}-e{1g=44&++Ns%^ z7V*4ivT6vFNx$8nW2KyR4JAqW^g8T3*anuhQ%Uu6Hv*u7x0Qt~*=LZ?RsBhvQ#;Bqvgy z?*66GUe>w2+i4Nk4xsmOPc?eUb`D}H;TfDBXa4@T%|tJmI3t=cYo zfArpNZJsnI(D9Z3jE-o_ui$RG%)TS#k)$+Kq#gL61_*vhltC6X&)XR9 zV!ya;OH$KjV;xc2{?>qcXlGsvZul=5F5{jw@dTpwd@fP;YCY?qucJ%90s92Nj$B~Z z_uoGOMkos0bVY+Nu_@V2p@m$`3@7u`&W}oBVwjLJa%H;14`<5|EBXYSaS14hp6(w+ zkGuvSyMyMoKZJxrEw?rZJXIU136kfG==D086RXS19zp&edv6&RW!Jrd!U$3#h=hW` zC`bw@Qi8xBDbn2_pfpIA#9$CQgwi1}ba%H!NOyw*LkrT)*`tC!&-?z*`EowKAM`iC z+&foX>so8yyWAqi2`}sDL(ow(f#=tOZA8*kVx7 zaF^)k>-yP+9A$Y4)fVGX@w(1MyW0`(Zc$!H#yLz%lP$I>8(oFt@T&ZFaPnR>yBER_Z)C>k6)ujUDcPI=Q)YY}%m1ab3+-<^1Nx z%1X**r7^;VUPPrFi*TvUc=th{MbGoyqM8s^`_^c-7^HP>g4~LnZuJEg!8T3ZDsf%L zHA%r#QQuYOX9@(|!ai48Eo>cK7dL}3QLC1Yu+zIQA z441h$T{?c^-6zWkl*#d7@X+@X^`y|Ftvyy$S=mt&&tU}bsClYOL&tnPJE!}$ud|v% zV`}ra5tsSA+*JwhjWq_zG&G9&31#FG-Z*!Ku0L6KXj>erqGS^kTCck*f3eF{4?WE+!wh5(}!$?7Wql?iwN~lnm2m z-+7xlfg0yFRi(1jvWel6>JS#knhlNJ`JhysZx+>xSEHX+`}4jRGmDp#T8?RLm@%au zvoW)CNU49L`N3^dX7RJE=(cQ>^~-MA`B<}St_Z308CiDy5vuD+&MA7*46C8@JfAY< z-ws;HMcHxWr}+152JlMCWV_+dJ0A)%U61x~T|d0CagMauqAOpqfp$Nv?yy3TEM90& z^~2XcxO% zvDV&_n5;HG;7c^p+!D5x>AS5^cmK=dRJ9s@<_U}|0&cBOf9y&N>=|_g>N0a9TNg3= zkyfns{kzX(EI7#2=X|s%_Qs!U5Ttbws$kHkJTrRqm)1a_2~b~mSnMLol=P*Z2url7 zUfwHbJG^U+=P2m_9&yFinQYNEeH3l6Ko23ifVykIg@dCR6%i z#`;TSaHo|MjGsZzAA%$9!>%wbcl+nX=3bd*gT)0IBMI*^=2L3d(o(zkBx^>qCa&y% zR>^TvVlq%n$qt}^nSBR>(Uz@MIJz`mdYkJ)%!p+1_8x*V(n(if*uOKT*wlQM!FF+Z zsA8mb#~@&+$|7jr|8M<2G07*!SxPZs`WbYM-(fOA^Q)}k(ada?%$V{5HIa|Yhu*A# zyLU)&6*0uj@>o)?xVVbWH}yRG-u#hT{M8a$$! zP>r7^__X6+9_!>sDSdCK)D;1Woz*;K2=$^`4tLT!x4u~WUF_5c9o7#svPnnzQfx){ zm7pKm1Z-yaS0xZ)tP^exVI0+pOt(MF-_jni;oOrG`Dd$tPZl14M~rpwn2m z7%@F|c9UwDkVW&7e07DGT6C_-)=l00jePq6aF;BL9%YWS-ogzEtS&!dZ23Ufi&?Y+ zZoM%a#&x?-j|_{9zh6sog)#lt#($Z#vwoB3?Nb?4OZ{xvogodmm2SqZ=_oHk-eu(8~GXX{@0zrG)uT^~*uvs_U2 zeUQQ8ES5;2#DhrBZ72_7t<(u~Zz-UTHoQ(qhpt)VHS_{v|7!e-u2hPW0B#Fdgt^69;6^d0H*!Rmk zSHkJB&JwnbpDCnw>R|c#x67oLjz3?Tl7JiE#y~PeJjx$adGXuNfxp94FcDZ?XWmIR zz7DL6rG3VjaAngCCizpgo3iV>In=2lAP=|~kAq%A?zd|siS%l_?3Fh+Th(j1QtV_t zjkgu7Ss%Yd31;j{@#JlSji%RAP6zI}&-*qVZdJHG#&MCLHeKV4`;xRi z{v*lC)wmO|gnMNL`s6w@o$|D*&BL11I$?hUsQ{;4jl--O{TFU-vqr~D7csGXSQAKn zc@OCKF^PeO;eDULWD`A6!vIj2CyL6|um=CsLK~Yzt`a-IKfR)ep%HY$s);Hlf+I=i zMluNCLFZyMDb8uunIkmI*1nwUl_LAOP($RVV+A>5FC?~R3W>3rQ+W38oit61i+A&( zo1YI6?zD1rK{nz6P=OC^GKIy(UiJAkx%KI6iI}IdY!XTl<<_uMY z9_aoOG-58eMnj-RLr6kN{MH}Yg9Zn}#(kjXOHi6ODfaDyXF+GUp=is|w!`~*2LheF zjPk_qWk%lmIWXKvaE(gj=Hh^4*kJy{XG0|x>b;K+{QlCML;{t)e%4(fzxfq;&Yniu zcIVrJ!at1xHnw&3F}VGljLX>m*8&$_z3IMXdE4mo>qu3iOE>E`Rwjf|cI1DJP9Yug zNW`?D!le> zx^*f~;;i{*Jq+u^7J1k2)-L8&wHI3saej5n{f``KMr>dJ82|DMts%%*C|9 zD{7LbiVOqf8JUI`bGz91ZcsfkHP;^B0(p@b?y0IgN7C05@Ruk-)^@3Yrnt* zh2Rd7j>%PVHUm%p>Kb|z@(6B*NBA#z&=d(9$kj#Yg*{`Cp^KmGdu!Tz)a%)s~p z-&628Eosa_18D>)_gZ}q2ps0SKV7-&(y5Z6^a&VhzGFfp_Cv^{M~}8fpC2|+{B0@! z=<7`^ksZ3GEN=*{Te@5)U0&MNukB=5wlAQ5Z*kkJ1Q^*bDdlLDOmYSOJD~Vw=7S{x zz;gY_F{8VfnB)FLyXE{90Z0_7!c9ev!>SY*ZbPulf)d4Fj_AZI0*46v#m6GGdaK9N zl9=&#qxBRZ?443KlN02q5aR!m8}yG&`M~*|d#0poiMg%7vqgDB1OqNZP<}!EHGXsP zjm=XpebNJPk)%H0k$Zd(#5*M*Q1K@ucb@x@b=G-NzEDk)K==rrrTmq$^uE}6+@lhy z-0q)zG!Aal=&5qHNpc=|cx_W_@ERE=ud_DGNIv`NBtV9f#bYt7RYjZ)SB#<)v2b*_ zpTse(4u%+okx^hL(H8jAe{kM_N%x@#JdQOwr?dU8>}9^u$tEN`2VNw!v$=gBF+3t- z)?#1@kX+ZW%=@#=GF382ez5*_`?#qXN&e4V^-NVCW6dSJxh!dZ>M}|2ROE^wUoNEk zVSHF*MK+e=8tZv`G2XV3M*LiYD2mRepffn|kB6bwitM zyTh)NT5Dbq3@jPgZjUberZ+bN^=o$xuyt5eQ3sPmNK&i~=iHEWo%pc#L#Ru?-6~Wc zz=XBy`jGwW7)lVvDA5~Im`7tbntCLXz&F5^@=q{>-|%UiC&7&OMk&w}e>?M)?qn5% zMbOCnV{+ybO!jXd1b1G-6tFH4l7!wC5%{@y*RVIBkv^~t{7%Ibm5#fOh~J23h#ZRh z#_B6cZanGP>rdD2IGn5~;7w2xQWswLy^@1%%OAKaIHZN0`5m0VF#tHhtH00mnc*gW z@}&-zMim*z2G@<$qUHkhOkZ#$9CLKRQ&&6aBo%BiC6?4igDFoi09*=;H00XT4nGXu zg1=*l1bQso_K1&WDCk?#$q8TjL=D0drzoWaaKsyh?3aJzv?z*6xD84+qaYvq5~0%I zY*7Di+zwj>!hp3B+j$M^9>n)NHRnjF)ou5!rCSqE*JbCNg`f(Gm=YGMD#6Pqcn0w1 z#8+D1v2mZYFpn1+Hh$IqBQBQuw2p=Vwn+O8z z-Lz-p(uI?aVdBJ;0IxH{BW(G4lZ1Zv_ARq+`t=4J+Cf4&p=?HNVC>f?3MJn=B<oa?K9KkrNmTu9Pt@#!zmH6)N@QLukNhUMQ66A0Y%-nF@DyPfXA}Z#+=?^^*rf3@p zy-fuX=PLHUSG8R>iA~INdfrrQ*0W@C1V51>(l_;8Ns-(%{DI|pkr>r0HG_klAE>4fx~(1d%DHPj$@kwT2bN@Kx9WKA&hT6h zr=gLFyx9=z*sq!Dwz=?dB+EJFOHmH=0diOLJT<@Zd78W6P;~zFt~rttkEzdy$J^h> zB=$4a^WK?tAzyU)g}Z!Oo=tu$$GPwo6|hzF@Wa^{)PqLkg&m!~Cw zCRxW05kMGm=dKk``+hKxi17uM|sb-k13Z$c+(KVkHDSGs~po-9zp4vdIrb z!j@iCLQl&t-!y;?t~GJoHfcYXg4%1d@w|B3JK@k{qe~JUUnC~@XJvslQk_@Ed8CpI zWQ#c+7gVYa&C=o?qeH+mGqV?Z;8ynhKY6S0ABq zo4dNcIiOFK;$OZ}yN}HIY;&SRFda*-pBN!F-cL7O5?H|yh(a1-lER0gjdjC&KoL?#b4W$%w_PCOW$~u5q_Tw)HmNi9w45z;{J{Mo>xS zwrpeIG9j{ZZ;@do{!ydQK#Yq?5$zQysWS`-zI(Y$ks;Pyqtq<;usM`K%*P8;@}n;v@(MNfCvB4xXEuHi?=dhX4m#@$!A;{=?~r&K8gKxRhO z4yE@~3SHM97n${>Ms6~v7uwB8S5<45Su?hVkl2grF?s{bdwkqezTflG%FdlRcHXK z+GQTgNLk&KnO77Xr40y;Mf2iDzYBE8tz3tZjpdPnc0wfSS+u`si{W*C^?sa!O^XyC zD+HhQ^-GT5JqC+oB91x_9-7EeQrkeEJMNUnnv5t3w?n~S$ z620d1)|@HT)uX9ws(9|f3};#BrzrGoX80%-1Po_{w6IV%(7Ahcl{zl;{y6^lr}_qd?E(~6iJtMz2d2B2 zW4#qf7_9P>*SgExI`t6x7@dDqw7i0hbNnk^a7yDeRixjW=}ygxBw6i{ZP+i^56?^S4D z4Ln2psN;ZcVxOaK=6M!rlT0p^t^kSez|8iZy@_Ot=XcE9ol6RO-Cs2mu5IZ(4-8nb z{a5DtDUQk`WnF8rLhaPa#;TQC3!(j`A_bGnZAROZ&xk5jm#UPCjyBSS3HK2v8wI?; zr>ijj>~DNQSHv;#cif|Q&s@u(35dbc8L8;i$mkDg-CM|+v*zgomsOTEdQ|9qIV#rg z+Uh}yzcLk$UyWgqQtvXQ82czeUSjg$CE?aA*qp|=U_m?W+Eb#D>_X>eqy%63$7muk zvj^PHetMLjvO?pv%777a63fml3YFN1GI>{!B-(8yFTORU6uh7@n(pti+{rH)L{pMg zZZ)=7^tg?4rFvbC2?JT2!LL@MgA)l1U9=>P>qnEP`_EqtdfZ7zJ}^yCGy`&qcR$k%94Co+?}qN^m7fod>m3f zcHg~r^i2Xj$E);k_tDPl-+p+AeASuUtS?iwcT6^#t2q$Tbq`@#zL1%<^vu;`pyUm> zCv$@^tXvcH1x&Kt5{l!Iq$Y@Qo}d3F)9ZR^?2jttxT> zBp4hu^Ym)oOR%z1_kHxgh#1biKv$Vt{k?MWed)Fr8t=F+WTaWvi(6AJf35r8=SliR zQ8!b0-#+e{R++Vx=XTY^(8AGHS>4bbRH)Ww?f&=m6`zhj_Covyuou^aDz0edV~64l zkQu0=Up&Z$M4gZ>s z?I*G=#uRAW&4)hPu^aB<+e!64i>7OzDD)WjWqh$v1c(2c=#SeEJ zoH@aHAi%}DN%Fkz4AvU`B#}?HUd=GdTdst;va63UUAmBt<=e98Mf*yU%jK&b+IEMV z15}9CCP&+g$*gV9+-Tl8kGq!QN!w2Ph=gQZvKy1wIw4$Bt|lyosAj1R8N7&DzbA4| z_6dz9SAc4g5gENLJg~`eLzieLO*ZDSav-~2HQ$DT@Ni|9=a-M3Ty~Q(H~?nX|egZTq`9)KeA|T_bSUHN(KY@E_0iQ#%(8Eqke7p-5dMQ zCStP!9@jvgnvlZ*Q?@pp#-9BUzE~69V#{ph!`fwCK2~Tv5$1VS?T36ljZKZlh{-qS zW32fjs2>eU`i?utV?(drQ#>g(`J<7R=vim131wI&%uiZVGpb_Jhd}1|=UtOZIXH4ki>xY50EK{;Qv;Bfx5Z0CubHF+~eFf zf5AeCj7>f?`T-_`CNl=IRQV*10$cM0#Ea;3+_VNPBl+yPO9nzmJ4Qv0i)u+YyVBQ# zEfrUdTSA$g9LHA0V{ksiKW>O`XXzI@Xh2AZvAoDhg}(As4icWJX;)Vyx}PG``2lmY zQbo>$*EX%2S1eVsNnK!w<0QNYs^8mOG@^NpN>eZ)Z|fk>F~w>swHjHvma^2#EmQri z&Qm$z`Vdqqbt`F65c`VI!K$b`q7pdfr25x&mV^0M9j;`&hA4JQVI~7$I#jHBbb-oR z#otT#sN=}n|JeE1L{~pcPF6+!G#)30)q`}KElD+9Fb=%o8BWX{d(EP?(|%e@(Iy=>=-yts0e5+&rIG+MV>aCEonwK*%|D;*ap+Zqxr zn!tdDLfCWJD$hG)NIA9^@UOnUeEV~2G&ie54uf!;f0`UWR22zASb5 z$K3sY?d~KNe3f*>M6wXQEHyTJaFNGnE5j-8xtwg;rNwZ(89lYQO1QfD%W=Uq zT?~+~x-TZ8@{*8>(n~tKDFDRSk9+9SB@;@OwK=MmIAd^zQ_MRF%NJC=zxBuqm%t-5 zh_J6M%5PQ0Ks~;Mvn7s)b#=q9Ytt^CGGZC{I(E(1c`%T@a!HawJi_}!9dfv#>Fof; z-cfO^^E{Nhq+3B}IfFlx0h(VE5eXiH`;xU?ZWWsbLH4 z(W)=%o%?tHs&^v=5G=~WTgFYqJ>|AWBWL83?&)BNCCTPipeXIf90!JGtgAFu@SSwr zmrGFn+BPM;AK3fC7B+BTq{&_MiIk3S5J+zXE508+{@Dsg*V6MEJ)z9X z$o7U*E(HW7cVSg8Q-71|0B2vg%!=e@#@p8|Ch;DRd$i`=Rsa?$t22w@Ul$7pi0P$g zw`0zNzv}4o9|hs-r;Z%p#wcPx5AkUK;SAtMl}I|DJxwX=elVXGY7WaN+|V>5=jy)K zK#-DS*WhdHl;FrwyJNWD-IgWsJe_^!ky@UJdIXATcZ&8nQYI|bVjy=bWgHHynIUUg zksEeqyNhcKcSX7$8S@dj>l|d~0&=;H$l4ZDH!>@3RH~`| z=!k#w<*gU+DJ5sDO6Z9iRL8{t#z5A!E`fY{JA!QH~C0E zGOjmCfYEXyv2El127zA4x_+qI5SS_(GYo11BXtC+r?6pfa`_qpq*qdq2Lt#`3Mak| z{uI|#EK^A`gGnLbPS>?pK6u|T4{&Io8q6BL(qUY4=ckhZ!`FSqW7hqVN47Eb!c`U! zX_XClZ9VuX918gohM?lD3}USQ@(>tf#)Eflw4v{1J~{HVplZ{pPB9IX5BN_Xdv;Si zh%GiL?w#3K`z8Kp7acgUmwZ(@6~)j!>beJ?ICXC2!A50I!o^58(w#~Y$IXsXE|2M9V(}i#8zrEVLfZ>zBixN=1c22$0NTdNVTtM zApQt&Z;yA^X146J6`@+`_5S>0=}esk1`QB_Ld^-XK}Gy*3O$zwnK=Hw_RGi#dsWhW zr7l16Hj6UrvC7We%hV*uNnkU71=k7=XnLwrxdoG~$Qw1N5OinreUI_1bj_3PG3OCd zl}}GcLmGJ=zQ=N3&kWT7+=lhE;h(VM4p};&8ZO^;$&cc*MJ^{JF(Y|ujE~BSTmfN(fJov8m|9m& z93O2*jbHyOXwz=Vs$SzrNV$IWOg!NPM__&P_OCp^%ERN{t)+s=2$87+NG zGRL(ibT_XpN}=3=Cnr_Xt#rU!Rx^_d3*^%4;vNQGRCImN466xnmrJ$_!^_bYxwnLO z8i=w*0B<*?8rJm)#4ER*M{RxQK`>{kj(dJMnDPL<+UW-szFk%fK58E3B7I$e==1eXsmdKHC1tp|yga zybVwEm&b(0LsFi$rTaI4c~DE+Xcaw{b=Q46ZxN%j%-BB>(*6V07PD+hEE}3Tl{FXZ zP_Px&vVt$fOg#kbsO<7^2_lz`yUQau;D~{%@a13p*e$@tldT&qUINXFQayVf-$Uy1 zy<*n#y!kN-fA3&UQkbQJ%fR!Q%u@#3H+bR;KNDRbLo)E1tjlP`kOFW9OD2(1zw4pk zI(Ro3rE-&ph#Z&8Vkwas&`V_+W(T3e#b7hc+{?Gwq&+^-gW$`MKjsR-Z$+3^hHBiG zUuSNpJ>m5K+48UE)6umCee=@>|GoR4UphgZC${uYuyFDTrU-y3b7oI-!ykg{KR$V1 zWU{Aqk;H&c;w??oO=Rm<_48yu`VyW>%vjjIaVuN1=<^l6C-o{>YHh_f6GASG2=YJD z^LJ}71F$9}k;)>X#C(bd0Fj_xQ0V37fu)QqzlVA7X5Q$MuxItA3()r@n!~RjeA{C0 z*tce?W+B;UB~Oj^KURZ@8nE`E?G@}pbU^Ymx}L48v;c!91B|8e&~wK#*KR&)A-xT# z{E=w=!v*zJk!NQy%`Uh7$#VYv39dr`D%VQ*_PP!giJkwfX#2TU=>O6aKu!`LQt0{+Z)0f)93Je=$fuOGP$n_$&yK19RYgj}FT+k` z0{)cXbTg9Bn*ow5yVgqmTPwo-Xdnf0Iwn^7Dkszgdc!X!t@Hj<6pTfFri1|cD{W-6ytP?plXcs}iL~!`T^k;e2&l$t&vB6HRUKCsRq04vQ zwjPNDX@V#aiw$+)5YceO@Y1zkEN+HPemot zdV0SAJXCA*IbE<)L}+*Vt?VmuHkr0)?hoON>oZw-wS`tnx*(|L=I7b`{AyGkS$X~n zpFz-r&89@#sn2HHdZBMoPeR@fUuEo5R|_BY%0*`<9Q0HUW?QJAY3Hu10Hxh7^581~ zkz|bYgIURonHCcn8dn6}xEun5uG}$P=+CjBy~kzqpsv1)&{X@Tl^q(c%|1P+|?tu}-0# z$!Db=M0Z!cJzzqvYsR;&hPjexJagwzYP_WYww4ew%sC4NdjXnZ8IkV=-nCHw6%GHT zzlNsIJ*@sksb^~xyf^K(@#+%Yx5%sMFTQU`a}Jvo9hVll{4lg%8ocGj%|qQV+3>6_ zLFCzc&!8=FkvK1qIpQHEtBO`Q*0; zujF>_g51V>mn7}b#PO#7G53wmMB1@6;G?6ott&oRj+C}(-O#PX$ORm?$XuLV{Bw7Y z57yDNc?mDL#$+?}P!^K}n`ML8t1*z8Cm}Ir8qE(Le50t#r(^m?U1%K}@406Ns6@)D zXbH}Ap=EVCVuz{2j~C?!4H{f$<0y(U38{D@0gz}P_qMo>c&RWc0ly&2R10j_I_PDn ziyh?1Uu$-NAfruSrP?S+l+`TxrOzEoGGs9kXggzf?F)yM0mI|#6KtIyyd(5IhjLf? z`=sz@-iMbBZvfanWYpIbAw!Db>8fyBDrkHy;X8L=O8V-qamGYw<-G>NijSPClT$+` z@g?s#6;>tbq>5|W!@{eO>nR>!wFpiK%K%Y>@1qr4d(Dn6UB~NS>MA|euJ-%gj+f;aMP* z-Hc3=30x2GtFyIAT7#J(X*F(}{8yhODsssS z?Ishtm=?E;IZs%ffm+v@SCy)E%81ihZaDwO9~kg&fT50+Sje18Lyc%3Sw#=?FI}zXhaof_DjPY%wgKWi(_|lldt$i*fY(>Zys^0o-t?RJ6b81l_i%7xQ2D!W{ZUm zZ~E~kw7CKwrGk?T_3_*JT+tDYkf&Lr0sFc4s|P zEyp}ay)nD=G?D~*?RQjs0aR=GF)M&7_40LRbg@Qpj*`Hxq||frr`sa-qUA`cGE-hB zYgd4-5D?5C0SxEWiNzj)wRADT1Uc_Qav^t&l=Ig(a zjX7+CKnK)Y(4ow&+g~fo-T8#Zb0{~hgoKc8hl;~S-8=p-YTUO2tHOu#?2*3bD=UI- zRDN~3t{2T-CbO*O#^wW+L;Nns1uuZ0^9(yu7AHCkJIkspJ^-bB#5we!B-4W_@+-%J z>1~60412v;*U3<6M-WExB>dQ)I3BNa?d|XatchhKk?yuGU_Tx%nqsX2Kq z&FPd_03w)Yby8om0iPFHie#>H7@N(dkjGoDRd$s1z_KYu_lyAlR2Hs?W*4W4R3zMAkOW2?R`Ir-cGyYTCRIgy&pI- zhB>qHjni(66^=mOCb%ogtw)W>#^x4K+AGo@W;l-A4jV|0`2q3Ev+FfrGFDx=7tLcu zxuANf;@v*Gb`43&4f{n#H~G&h`^v=h!@EBw23e)P32z^HRAESioYmU}|M`jV6Okg8 zfrqiGzwCa-a+q>lnTurW=YE5tR!$=)Ls%$|q zyp40#lyz@<$nspGoFJe-=8W1jx4-Bgbv{g@tT?(NTuP4KHzd}ZrpZDG5kQL2kSaB1A@kmfX!YDA z)}xaT)%$`?p`(4soI&GtV7qPqe!J-opN=%dYK{okX4R7SrPQt7gFRYWK`z>{?P9yGzCZ?QEs| zVW9|edEt4ed6#3KB&~-r@8U#VUEIYEeP0QdDhz*#dqilO-QX4ffvFs?qk-M>`ar{9 z+2igD28J8mcLuJQy~I+QZwqgxVi!D+S1A@A5c2929oc<$89$tUKR~8oM&hNZeu0G0 z&d6o9v|{b=Hb&K(qkC@hPJ+ig1wsN@8ec9R=Hk#uZ>Jz#mGs=^uNPU5@lz!$u1*1V z^_KkIAI}LlO!Ytb63MGv|MFWl6JS^eN#++3q;i|Vq!?Y?FCn-&+GXdlVJbl=tcv;6 zn0xCz6?Z0-#w05jEocNBu`1Tv1rYp~cJrJaYcfkKx=O8!rC9cr%m;26eIAr09rbrQ zoJKwB!HoiZ>GNE#!+pgkQtNHZ+!QC+0xWtl!17ANv++qBP)sjkI<=xO&-xsSU2lEn ze4X9RJg>MXo)^ljr>m%j61fP0tt`waxi;{_D=1u|g>4(hNvCS8P-ErC!mAQl9Zs9SG0k9my$?-MVRSV;Yf$`tS`5P6Q`zIGw(Zo+p~zT{doFMl{~Jv`M% zW)bD}?&3_6oXWSHBr3I;Cu;s>@4RT6b7vC+{$M6=oW)Eb@yet z8sAyOEOd^V{|F$kLu2~ttznBTtx^EbdD={~VGlsrfMpmzk#t>{rAf5QvcaD-M6e(D zH%XFRI@l!Dj3Lcf?4fkSWU}YyIgVDMjf+6=cl7DL*Ve?G%~0VdE)eT}Fa2Tt)UhWk zA;8X@HO~=gH{BF+k-=QPShh4GYdJ`8D{^_pqD+6WZxH?Kepu)GhjQu-OanAdIA52cIl0W9A!gA zCcc5y4yJ$vSLmWb*MivF z#cJ^1po54+1Ao-L`=NR=1IV%o&#E{+y?Dp-405riG*4GW*1x^tpyV}?Vi=aB{xk3w z9Ka~9c3hQIxCQ}r3f%a5A6WpM%JU_HqwE!#MyGM`5i+J($YXCXo{Jw1n@a~0yyBWV zx*+ElqUIPPRcOr1I-0CDbul5bOPBL@?4`YSzWSC*J}TbiK5pvGP)=fNS+t49&=r6C zyt4R`s}-@pZ&n{3i(Br>w1J^Fz#Mghz5Y!z|7}(E>1unj9Ke%IlnlJeeT&Fd}sNry^>d)bhx*9JDOgX`bjvuuDRqqh!j6KJ(ohdX`IwP0lMM}AgquLMY~LX`aQ2SFdYU3`bnrepW0FMgaOTd_oz-+%N`8lQ zaF{5iEBzHHSBe4@P82%fh%See)5xHb>`TMhuY~tbhaMkv96!Cs?*k}>&Kr8wgX5-| zv*Vz&=5eou{JJ626D0Hthya#^NVybS(lx{zqCufhG>E$gL!f9AX;n9z?9Ux$cyC#y z)&!d})QDd%AV*`rxqk&8w^_P@W>ZKd0M`p2cPw~uG-E89p3BrreaJ3R?-yGXpbLwW z){jkf&y~Oz;lOApzN+Oc14tE+G`_;HVPd#X6x6D zpQU)`w!rK@H|*Pef!=;9i^L5|hl9iY;) z6{H7@Y`ULydk&vu{O9rJ8fs|Dv(iPy6rMqwbw)l_dES?2+|B8|jH+DBEwYbB7lu9t zGbj}e%)M<>IiH%$s>Dd@NXic~mR2{#22%0Us{w1CYCqe)Re5K=MdP%v7M5_zRR*ZS z)@7MC>p^()7*Hc;1n`m>uB*df;z`2%jtj=2TGB}x-^dVRJ3oTJy&UGPF}xk=G-Sug z@9sH9Y9ss79-&htyqVFjl`u&MXt1>%&hWde8jL)X7kTS)v5Ix@#|PS@ z&cG{Cw~Ss6)wq|{m^oq+Zx;8Oz6A_aBuMv)Tgd5oDhoIY^)gTwG58M@R63krP|SE2 zMtL3SOMw4|Ndwt+i<0c#?;J1uJ(=l1Zd{F!G7d4~9W|F}a2(wYCdp|f^9+wl27g}L z$_FbMrB#cOI@G@Wq2;B_Ibq#7<{t}W$c!mLm8rBWsK$|;Z{ZU5^MOUf1k*(_L(d1E zGI^L--6WtMlAMZ)>4bRjQC^{P)vw+=Iw2(obPchx>NV9S_Oa2=WNhgB-m87NmU zd_gs27X8f6Vu(@mev>Wz9zVqv6+$;;Sth}~NU08w0_E7l^-6&6QyNc^@S1Z|X-tw6 z#u8CtFg|m_^(L5NMP8~gmAj6kWDb$mPoePxdGYMdwo(X9rYLU!r7aRef)kUtuFU!- z*};5g;xodgoios*%{PbH36h{1(W>eIGuXJSkY)=n>4Rvx4`vY8K+mjn9T%iE@_d5vt)+}K-BQ}R zQ!LAwJ~GlKDO|Y=&?$~RK}AJhN{Y^|O}4*4l>;_7#qqtOV(vog$4th6jxNt zXBbKdvPP+fzJs6SF2E6kuJ^nXPm7k%A(IsM0N=RV;bdrofQ(f-W>MEk&fe56ZiT6l*iJMS}4zGCc2&XYtDiOjpuwi$oBO6##}AQ~nS>=@*%W z76a_-fWaB;_)bgdxDKex7&Je?mB~JsIJ0+SL|xDIQ*Lg8G|0$}rv3$)6EJy?5CWsY>!ev^@hTrBsN5np`h{%g;qbz$=&D;2Xe z0)`^?@}@_K>kq;Q)$hjCdhCDJ*7^EKI?4?Y57BAw+*(oX}0EC)VJ-3*{W55y7{%k31LDjx5 zqc=x;AoT{JbO7fF)1QiXXs|uV2i;-hKv1uFmvFV)Gt;I(F5_FKQ=}+KeT+#n1|etS z)31BZ&vs$byBxkoKb+S4CeA-Wd!XRuPGXhM_syQf>R_@?DY8&m~^RM^5y)K6DGA zNYWI_7gGT~w45|NmKra*KUeO7q{<0f$7BhY*5in_A=oE2^-#d>6&31l(^nXgsUntA zgYiy*TF3Ndt8^<+75ivbRdGc+rBTUSY;o@8Xi9tc3h(F>Q78i{8<0tbzAbLQP>ZoF zj3{q=Wg@<0Z(X&(y1+rFlBTmtiIZ6YS@Cf#id=bs0h9#atUTY@vM`+ZG8 zAf95`cw+_1JD2CeUoSyB!sb?9v_Enc!ds&GJ=g&B%P16MKO4}96^!?uRgO`c5rw(d z_OU9?U;6gxt>gM|jk|MFv+}cG{0Bsa8^~W6GuQ?*mtx)H?4VaL&<(8JrB0sTdd@@- z_UeU0DSm#=>$A-DZzbL;lXsdDr35i?aPYcse=ixa5cAVpFw1Z_SLshE{g6~jp&e`a zXXXZq`+fogQh;6l_?6X%Qzjl9#PES;m=;*KKt=~sWj>$9HCFXjglf*D*rB_Gb7A%R zdjtJr1f27ouIHzRdf^UOENMj-Mxepj-rX1>PK=st{i4t+^hvpc{YYIy^Pl@AB?>$C z0+WK3V{9vYrFnY#3I!H);C#)^GA0y5u;Z6BPX5~P^t(+gy^Ag^)nzd|60(}TSsP)f}AXHxNR72ZOp6=C7nE|W2j3^F;GFpoT# z(tE^>P9jbWbenKd{8LGT+Ok=RUfqwh2Ms(!D15!HzQb=+jSgb-TF&9BGXH{#rWzA*r- zy$-?Lu`W16p3{Eh!=M1@`2=qNt=~}y1_YZ!VO~3r^g!_m1~$heEg7;N_qql|yVUZQ zWZ;!sAo0o#vO$DuQ>F+)H>RD$d&8_p(4SN-z9wwo`O)J0dqU*Hfv`*nK2~?gbye>E zQw<{_0Hq!U7c8b>-mea?K-CQ>z?&#;3pju0qLU2d0suA#CTkgRlF$1{{$$`e);&Y2 zf@D5yTyN4@*O~O+lrZ!Xy4J#$#uWRcasi!GbAe#p+quoejhRw!$Yqle=NJf%?frbh zjv1iCh~XuI6IDkGF5n{dek3UpW~GqGyzxJ3JwT~e%Xk7 zQwei*tr)6u&Wl^j0Xf4A^?Gt<@Eu|`1#jN5NggDt_^!TCb5I65u z(dhsSkp3S;r~J`}#=!d+>b{XFotAQd1;HdwzcN|c#r4+=evk9i-d8<<``;TTY;Kjh8B+E7erTJ%m^RFL?jX_@`LITC7C-MHy{QlmW zZuG$v9%`!7>ZAW`5cY55^j??Fd=6Ali`QepEOhIN4C^mB30#O-G|L_%MkJ^z~#K>u!y4IH<5 z8jraZ1oOrMA#SLh1;6w6*fk!@6EpDdZl}@Miam4)2ZDXAGx`3-Bz2bSarO6EHjDlo zGykWR49*2*BMviz=i%4Lq^_rTfz)sh-~_IPP3Hf1(~{5$rbN1TIr$LRSOXfip_YLq z!#OIG`F~kh5`dEKOMS%04Go6@5)?RLt2g{}DiDVIs!h`W9{+Dk2RjS;FeZMRNz6nJ zH};{T=6ATsZQjciE{gg;-m;Q_iyqb2`!m7B>FM4+P=L8?v#ye<>Clk>cXJHG0R9PU zlD-oQ3jm1}X|jtPO^P{M+;3Y=LlUSdmj5cy{^N2m;OJmmNozJHNgQehcM%D6!F-eK zTp#d%%_AH=kIRt@7noEqWdn2XcPE8`CrlRs3i|Jl1QP*Av#Vedk4x$YD>je>WSlI> z*5Oa%nlk+NM?wo>2T?%}G+&~F5P_9OAfE5(rCyMIZut3T1~?Zs`n6m%)a!w}TQN4W zDJ@~$*)Q6FFQx{EsO^+%%SL55Cd?LE5BTC(&?t)q+L>l#QngtVp|1(o8YKPgPTr8E zZ{(=I(T&;!oc{)vp6Bi(aKNI!6AtQNy17g{p&N0(l_2?A!ek@I@oL>N!{!#X)Fb8o zwu1V1!`!Xk`MK?@)t%5sUP8m`1wXu`^0}Jg*nUR~?d8F-YJjk|QA-!#J|qg#;6-|7 z*`OS z@`h9a!?@SJBgV})h`s_lYeqXkahX5Wf&hj2W!u#Nyf;hXDdgz5l!bV0r~FnbC_j3Q z%UrA6cHw6SSYqD(U1#jBnVyA+kfrNZm70x|=2+3J!?8ARbg`oW&<>=ddNVac4ugJ& zG2oYIPRmRV7oCkf{(soI3aBXCZc7X$T>?rBAdRGm2*?abiV8|fw@8C@4lN?x(h?HV z-6Ksf%IWd7t+=&pG?-y^rmXt)2TF!qkY+zggk!HDGUc ze||-eBF>ad2>p8(RO!uo{N9^IoXPUg!{g1qtDw>UWKU>}$?~+fe7_1kUP?eQ5$^fR z@FJNDf)b9mzJDm#J;57%>~!MN9MshRB@#0tra|PYTio$O!d@EQaY+>J^Jpd9o*~r7 zOr=lQXgy%7Slw#UeZvES>y&?00!1hqY9+FEio<_19UnmUs3BI3_$V~S@xl*pb`7xh zymnhZJnA-B%0||}^o7jC|K4QD2h?WLKBOgsqRzjs9))gZ36qve3HAc>FGd~nV+1hW zzuG)p75977jqzSO|H$y|wq2`>KPA{MY0L%h_RYVHoV-+;fcZZJwMKw{?!S=qzY#Hn z6j9TQwP$e+MGA{7mbq~cgyEq*evo6~_2pJrHgMyYIQ_e2_{Y`;&scl;jY*k>_|NU$AWuTn!5Lv!Nr@w-JdnOVh*k!GF`z^|?rqy(hcx>;ADe><&deSE&VeY#@mR(r5SAU8WZl05W4Kr>+ z64EW^bOL%9ii7UkSDh$Vm7=evVsQVuLH{1v%v|6T(CnJt{yt%0TWAOUF&J%?_C~>D zgmPTL^~)Qhum1L_{rh;k`Iv>Bc!fNLjtIp(Q?i00a&-=#XZW*Af*oZ=Od;&qDO z-V9Zj&4*Rd`I`O(F#dgb>%UHB{~Y=H_!wm~ZkV6`%%11h8-5;>vMp|Rn(_wIY3G@9 z(i88SpC*t(mOw!EO8URzV6IqH0|w{D3PKJK{iH}R)YP)f#)}OlPKj%Bgr^34I#rWqP z$|!(Q*EN(EU@oGVt-uX(oQJ^4BRWg9IKz0nB*l;gjbw#)0y1Pm7i#j~qvG!(3R2SiX9 z=78EtEj=bPX236vi7FFS4f~<{RoD0HZ4b+%rt`K^^KeIspaIFy=V|t$UsHUjowmI_ z&PDpLjI04`vs;#jH?~74Kgnuq(t4|M+L`IzvcUFv`+Tqn2jJcgE*3%q0;z@H81}}q zV3Tn~02*XFxD<6WO9Vv9x6RxyJM~$?rMf3gA z0`K?K37nc4|9YdRI1*}0U>F0Eg%9ZAtiJ%H@#lbpyFN^-HSx=C2?HOc+ink*%2*T; z3Pk^1Ky;l3`rD7!QP*Xm&3a%Nte3he__u>E&yTx6GlG-(;~`i$X#ldN>9z*&0Aq;C z-S2l74^{_@fKf#3L$f$a0qd3rZT3V^6P;OoXHV~+_3s8NWRmuBe^6q_`HL14M z;t2OhKq?Y`HTG=6`gcQPW0{y7BJJ0SKw!GHcqK!P$Qx#TtA zvTCu=?W11lvY0*ZqxovCO|Y0JUF=QKf%#B_SyImQwQ^?^i=}ulo3c`3O;*l@!2nUE z^`UdQ>*?Q1nVNuv?#S&-{iEDM3wWY6`DbH&^8k@&vVUI{?kn3m{Wu^g_oaipf^a zR*(jcnaNsU(d?<*R{CiZ4oiS&1i^*E8Cp{#G=d##e1bC@4F7$5|NNFSBFF*ItKz+? zWt!_^DNT0=e#gt zgLK}P)4iN~u-VRyu4#hnJRull5dj*2j_c>CQT#9H4H8X*5$Znx->n#Dd ziwemOw#+&@lLf&29lzURd=98{27r&x1AIIZbi%^IQa=rX)nWPY;X|HSSa^78v_Byx zt;3AUz9;4$=#^D>Hdo>;pkRd{{qGkn1?(1*fVTQi?dY+Qm9Ks6t6mImJ!U7qZ60>n z)>nm>ml!eg=6loLw{wFjU*T7o?-(e4q-RZ((NYaAv{ZiF>sd4X^J~dK`T3O}nzoaj z(7I^*I@{Tm9lh`$Hqou8PePLISs%(-{vq#=#q=UR1c;!FE$||$Kw~4C<2l-F^Rz6s zek2aHkP2w`JxQoU6q%{2s%i$eIRY1#Y^~VX*u6gYtJg+e$q=#&u^D@>FjY{$_4{tL zt2biD(lg2_ef+=cML-VLYs5i^#ZY;21|Mm2Mtkw3Oa zef!q!X+rM{w7thBzHQi1cFCmQ6C`HtexW3vNNZrqt3hVkozEa_++ir4rK&p4pEqA+ zvLZP>jB6nA9k1L@UGSeklp5FozL70OQd-)*8?TuoS(RERKJgId>fj&azlx3eIzhu5 zqKm;JEN9oLmD0Q#2Z)h5tpR}b{o*6pD273)ro${+tbk31%U%bQ_12_7(lU%@hI4L# z8wG(zLGREQ?|QN`+Z}IRMWHVF-w&u8K{944RPkA0^z`KTvo$47X6YFWL5;Oef9Q$W zL;CaNHXYQ7Os8ByXYT0>$~ve;kNOwc9~di7wA&K=UUYVaN1tCbkA0EP`%iE}U@{;z zqW;KLk*iMh#FO}YuGyM8jQhdV{RbI5GopqB=k?I?OSn8Z7G z;91$A57>_Hej?Ur=TFKdDw)v+tRfa?;C<7$m#Y}#KAf1E1ZMplP3qwyW|lPnI~Ib_ zh;Tt6fs^e6nhC?Wf|or^?u?QNvz;wIk>_&m_iRv#djL z-1F`bg9Dcnm!>ZK_jCS~9~lQ_d0}DYV}t4G&z}myDuDyAb^-(^iWda*(w7!6_0386 zpaUriiU30>3C88&l^eiyqXHb7XaK3mM%#H|>Y`B7g7b#N!5N05T49+og%Qi_u!V}7 zV;@wR0h^e)x<}9h3Ayh>WXA|7vf0=7xs#p^3d%^nsb4-h^-5yF`Qw`brnkQ%;k^_W z0pnzuSELNN?h#IcPkN91j*s6!DmTi`eI|uYTb*qdHhj7~H~)So^L>cyZ7CO*nqElb z#wiecb%K-VMkPlAm+ATx)UeLwiJBUK8h`flO}&5l#1Z13k?EC;ouf9yXRx0Q$JEzbz$fgr)A81~OON<75g;F_e7_oIG6>ic5`-<{UfBOR zkU+!)+R3YHMU?}arw5_ZSfl}y^*nBrSo=06GZw$LuRm)gHdb97oO*5$9P$_AZ1z9> z{Ox?dt+14eDqw?ZjWX{@Uz=Fo5CPz-6!!hXue!Se$vh=HJnL6??i+Ifw`A_Q# z;nnBmq4K{zcLadYJDXA4$+n;{i($p~pZNWh9072z4&aAwn-6fjg*7U4j8F>CvAh_? zmxL`ozS!`4+J>*^Rc!*6liW3lh=?X1!gm^w7+SC*W;)v2`}^J&v|>bzy#v{;o2p;| zIQ<*}FXvm%B!pt$)p`O!Blyjy8H%frDh;;ywXbzK}Oe%2Mb8EgtN4l0{Pe>)F0fqyrm zpZq&lkcmO+)S%^JyMtQC$kjz1OAFfZKZ~=u1)p$m^qMXSVnKX72ysHNJdJuI+IT+A z^YWM%7ZpKlPiV(!u~EUPD^`PR_jz7y*L<74SLf0^Iqf#R3uFfecr84;&8exLwn3x+ zF=m;|rFJQn?h_1zYBYvg*|hZZ)elHZywlQu*bdg z{a%t@;=j}nGaYv>>9^6*b5R$#?=)&}nvOHPD=P5q!^ejmD3MgwR|5u@V|AP&e?R@w z#IO%Zz>90mT%{N$TQ5h1O^0c?ZZTOQ|M>li_QTx-h^Hu$sP8UNyQhXTy{sz5E<{aj z2AZIGC#7F@%n1A!s7s`*3jGX`Tu^r))gf9@G)*K*&wFyVLf@F{n(6Tf&C9&T(qhL2 zegHFOH#q=2`?Jt4ZmDD-C*`qTkcwuK0t5`*5ERbXf75H3sGH?8yE|l8jaucR`01YX zS2-?vzC2TkQ+NA=(6*_#)iMj_CkLjnbdQqCJ(z9432AMTA62y6ZtQ4pm$5)8vTkOd z$Mjd?iy)%);1MPF;CP)8_@&_o;G%8TA{9a?=s{Rpco?}B9%rpmA%&wbAx>c0=9F_S zDUE8?dEn6pL77+~B}){9^b#}5T9Uv3!JQR|-=cwc|4Nd@Ff|;hvG0NY?Qa^@N;9aa zeH0R=#2%bDEnK&B3oj{m*8W*DnJACrA4g%E$|}7<*01@(CQab^6SgW>J4xcy%lYbs zo;3$w5Y+2lwR6>n)r%k8d%}*FawDHb=R26y#S_WI{rtwrIVlLcdT%fCfsQxKof-=(;L&X!6SvDrSuqO4C)Uqa2+9bZs*2`m?Q9L*LjK>lQ4m*N=6^l^=wveV274>3Zm;R^li6jI|_?t;@s~d z`9(>#a^7JRt$*b!d^VINt+Dzm8}Yt|W9%DeMh}mH8x#bgNWrs`s9l?p*rD{Z%dSN= zMV0~~*H5(yOE`J;%vMKp2C-GTpV#|8M-mEqb4x4`s??d)>(f|r>gjGW)L}?tEu!ff z7#wPrJB8gl-#IU&w$M1T_3(KJ%0^Ax> zwZ9m2bUU@d_;3aQ!~5M>r#KNaHL^dY1mvdbm(*jn_lB=a@i0l;MhFLb==JFrZnyO{ z(psME=)?-z=c(xtbl!MXJ_8y5~))>i^eFK%54^^tLM0cO^##`@b zQT$hTpLcKwNjfyP2-DKS0r@0@Hx*>jmqd_VUaDy-@4?DKVJ0TYdO7nvper2!&LXw- zgV#G$pj7$_RIgeMsHrsGX+V`)ePNkf_vnRLjP0YOLqYHDAjO5JA8Np=t>Dl^M2Tg* zi006^bxDEr!k1!-Hr|s$6PU&Q!09=ytyPQFNdlIU0fT+_EZWZB1Rpixi?V2H2XtUK zAlx^{w<7UfO4~O7IAQN#wUl5E6ezhlBCmh)LGpyD=9U}ghWh#05$J2gyv=eEsCt;g z`Rt+K2*$&6v~^***i1GcMAOZ=|y*jj%uQ`4`~O+D+4IL2Vl+BuhpjpKw|~* zE5{o>1kggHst3_7$8u!qW1Te%%68JLhKSPj!u3~OG&Hi_u+;4&+*kW(oj1OAx|-HI zRBBx7(Q6=ZpG-Pgfso60rGokA;IY!{X{R41!?~fr=5^{zO`t6eM>q@$;o^FW;uU$s zh1x;h9T?G|T_DT_4SZxZ=j}=?;^%=>G`dko4Imyo6{Ouv6B@UZ6zUWHbHINU1(}E0 zlgD-WTJ@S4vaxgHzKE^qizWWNz9Tf8v-vo=#ICc1f^Cdl9e|Q};t2cSoucFAeGL+( z41O_!lfk5`G#_*>G4vi_TCn^b-|KSfDZ#-yxVEUtN%i1C#jIE2Aa_cA#&E`_Fn7Lx zQ#Fl?4&xOO67D3P0cE8yd0J+svp95_v+E9YwtfJ!y^VQ!mBtd4Nq zU^@w`Ijlgf#-T7*t^5Ua#{nCY7Idk?frTA*3kUkY^;ZiWBIY)HP01#y_LC~FARcM= zL#;9f$-LJLnRzcQa*Zy(_Jq0ac<0I$54fTcNxf6dyJvQ1C``leWycnM=FeP8ElEf7 z{I50AmRH=uiobN8DFwLp9aaa8Do2j@wFjvd2}o*9e;1ZS%h-=epn0o;F(jCU*n1mb zCX=R?BU4`bI`QHCS3=KRo(`6JXXj(QOblPmDK_XG*RZxZUK;q~cqx}46BPfLlb6S? zeJM#3Xyr$nBSZf z_L@B8`tX}@y0o?JA%=ISlHBv&H^pL4R11D=`(eA=9hi>6wrD1&@{zB41QHjCFP^AI zZ2EWOJ5*ANILmWrRkGeaqWn>OiCP)>#fYPQ>dR^Rn}$10G7nlot&JAT&PPP7`vOv# zqDM?IPTR9-2gd|On{w}0BBS%5gwWD}9_mP8PYd3}(7xM7(!+IMtSzp~Ue(24d&Uk{ z?&Z00F8_4w)u>_@SX0j_#($SWye#^m)QXp9q2qIv45?xqD3%MQki5(dZy##d|6+LI z)u+V4EWJ1IjC1wX{dQ;yrfz!k;v50U7!9rTmUxSVxn_(LpO={Mu+g`Fow$>ytOQ?*?*gVkJ6PbKZwiP@x+QdKe98eL*I>1Ho2}kk- zgsjo+`)*1=P4X0&ziipbY+!SvYl<;fqS3ZN5-Id!CxJS^P*G}{mf#bO$dnjTV`)qh z*~zJ44<7Q!)~{;65-rpCf0vdn_V>~@_HTHhsX~W8qla7TyXXFL9IYC+Ivv)$Z(w=; z{T0iT-cl;j_QQAm50_6Uz{1-2MwqmjV6T2!Z`ZgSXV&t1dd}F^H_bM?G}CdxqZuR^ z(G=M}8Iw@?T8o0`vD_YWh)VH%h{^Hyj_f>()a%7Cb&RNflq;^uDiOP%P{qSQ<$jlWAzTN)WjzeYY*5oX^U4Go|y-Te9{X95i>{$rTqJybYxz zNM~eCK||)|6T_lMok(BnyP&~pxr2=Jh?c8h;~7$;8-Rh{oQ#$jhxbAe%$~M5M40Hv z`eblOBN8J7&18&<-ZL#N4KlCLYL1HfnOwURy-som9~RS1x^BmT5#5j?hE`7#7bP#j z{04O~j%x>(7FichMq@U%=u97a{uThxx};+_U@X9kZG@eDG_HYaWqqj`(aqqJ2_Y(l-OP{6qK{s~de~}HfK9Ash+p&y*d3(xA+{qG zO$*Dsx@j$La6&QtI(6^D~ZcS3LE#Oc6v zfrExr{q9qa1H#3M9bT?5nQNao!ioRnm^YcG;7yd(Oej)!HoeETjTDhDU^@kJa@6&J zxYNyNNUNzeNapekIW-jbq7}dQ*ypIOd_JNc^eWCUJ3sG}w2cfhwzWn7;ClCo-q9(V z!7V&|6ylgzH`N~R;B=2`PeQodom;&7I!c=sD$jImmCVFnLkDL?^Ny?Pf4sk+vcXn2 z7#zI9KK(UJ#YXWq8Om|3w6)@W^%YY1nU=lcJCkbGNTU|MNqTfFa2x<{>B_atX4B#6 zhB|Us6T7Q^SaRKCQ0~H!`bZO*aFBFoMp!+`Q4L}=CW@bWT5Z85ELrUhsN~WyiI{&h z9QzR%;9?qmkD01^5*KxTyv>PDzDAaRt~%g~gyA+DJ5Rq>Rw`8C&4_wHk=wk1dWhe~#LS+VmZ?)M~8@%J69jPjYAC z;!#8|LD|m;$hp~SOZ~t@u$j(tv@598Y(goH&tz#>(Ng^TOMu)RV0)5y{k!jUV`1F^ z&~_AfEF~r;c>()cvVqK}d+;MakwhY_#jcJVe4MDZ>BRJB$B%>^+FodZ{JxiMRotdy(4-acrgQ@07FBjArNSXo(qLpXwdRGjv^GQQ9b2Nkk|b&u=O& zeG)VUkq<$1g=RVGJt{pX;wO{$hqghx2T|QEVRDmVkVThk( z%^IJCO=VtEBJ5ngH>rO(7W;Hrf`W;>nZ7!uh;@F!(Qx^p%eU5Itf`8!fVJ_ptc{G5 zik-_*L~!pXjks*tESqe~VeVhQI=aeb1a14+vUuzp0~uMr_|r*oeJJ|&_MUds+~PB)(iE>c9+p3CVYMworkl# zk383&h^ObO(WQL)wY!^LY+#Z0k(b#Yu*L&I%;~rK-g1q(F`ujX!}C~e=7T|(ZtThoWBnKoDKRIE^lFBN zTU<06+ah=RUjVJa;+d%iY#3);6pP73&5#CDP*^QP>e%qmwW#rU*?vt`0!R8~`1;`8 zbpvuA_cHzBjmOq99~GR^&uC=iBfYOX6I|80J|=MvQPOtnXIsDBImI$m5r%bUFMEPx zRRSqp;Guc7<~2YM__ndJDQDQY4XEC#2$6q~?QGhuZ#~n}W$_bG-z2+WCN!1X+vJ z1k%*)q@LV$;#dldT1IrZg0DfF)&MB%gQ(f_s4D+xb#TiT%VK2Jj6B(GEn#f$Y$T@y z$um+4Pl|c_iI;HmoW<zW>0X|f1TwY**(p7V@XnktlY;Hvavt%!d7|z~^(i?%` z$0WOh{)^vR#5$a2Bmer{Gs`~B4Wak1u8zA5cr+sYt-+5&hUe8+=v_=*d<4oXZ)3j7 zXQdYWSs5a%_bE=A^czicSs;7#QC6wOvRb5TLn&ph5B*5d$9~N(ZxlU;*e7TaUuiuk zeth>PElxzmrSyVpd3P{lmEdl@M0o*`{f^=N&;)vSuv7i)gEjaFv@VByYyHuH!)MeLD}RJ~ps-lmQLnJ|iuQhZ%nM#ftALzBCJiym5kb(qfs z@L0I&p9MyU2RQpWAX&*WJ3_)$)g@9{vvmDs@@9_uDolfO^OX&cYk5^xu6J$Faz42_P*~b;c1CM`I6N6B ziBA)P#SP_is9EnlZyNd9cud>rQCEQUTHg*TmzZ1HS+;X~=Z>S1UB+$l0{jcj5V*Jf z_;qG|LQ7Lk+Q>wcu0 zw;^Js6qSQRs$Y}LNigG<6uKARIo+y&K<)y>dE6(fZP=Oy41VR?5_qt2xg7-+pV`ap6l)G9o+<%~qw zaw6X_8b|1aa%75Ya$3@#B>djU!>&mVEstL&z7n}7#KW_bE3tXGeUpPOzYM_ zBa%z*b^r3BMTxc9`J=W;jfBbiLx+fo-w#WI+bgscKBPUV_oew3j-777Ik!QDDK3tu zd!7Er?&$)_D@$dqbw*)W5W*xTQ*9pP#|!PN`$g8D&vV~LkT)UKyoRY4UeSG2JrQ_E z@I(m}1*Anh3T57$ge`AwVQWH}jTTATTVg7hyBIuRB~@8nw&P+g8}12Ci4yPYUObWS zYhE*9D63Pc5|3kTMDg?$w#&*5Q#H9I!DVYV9{nmf;_Ek<jJI2CHPliJXnTG2R!;nNvn)O&AH(~C4+cEy7a4Sn)A4MpnOIFNCW>^H|1wUIvU@hN%j z0vh8g#nj>Y`BnLY2naVe4TD)O00Y zmWpNOzwU6S`n|pmrMe;AbO92~>~1mYIeixf=LIDq|5gjZpP&N|;uWW7^2Shj_J*Q0 z-^+UR2=Y7pzpmK{_LCc)>AY=Y^n@$ou<9XQd~sY~Ea9n;_$iM%Jlp^kHk=vBNlYuo z?0%?kJ1J683&N{Rr1hSKgi_yamL@tXt4D&mf$5mL5hxa|0gv2!ys?^Viq19N64bT&k z+ZAq*7Ama~B^AMxabNbldy2QaA~t!peG;}yh*FOzd&u~^-P%ze=Wp@y*vjRZE*_zx zB#}b8f5@-(vMR*QpS<|pP?h6>$T6ZLl6tb<*_GHrZ=;Ux+-ZQ(LWs^~4=oQJRtM7G zY!oC~GKO_@eIUgDV)kC5&l$xY|HtN&rwuQ^-{HD22NQza zrf9XH^LTXMK?nciLN%fPY#Bdy$GAwQfqRj7d{A>0tQV`S{WdSdf$Zl;A~n(>tNuyE zJLD+55rn)VTHSTCE|O7QdcnesV`F3Eue&;u&`0*R-LKWa>=zb*F5?>=J8=Aq2KzIk zq`!e6koGP%YpIOBa86bzPboG%l^=|oo!BkDE4co!Op~q6Tks}a&)`0JeqJ>kj` zZB1I+Gz$;+X*AJp*eBHPCK-vg79zt}l`iV4!jFO}j@pSHt!rnSiDW!*|L!3yBN{7< zbMnQ37;Z_!v8eImQ3x`URkNCkFunQekI4yhC*iaH{O4i`#t2g8kD&g}G|fKO*K@Vdtr!MW^3)0MSp}0Ana(;Zphd z$_x+(^1?;cVU@X{P1X*w-DmpfB3B8C{1%BhFI7CWd4Z=(E#4Ha9j|eD!YS=rNPLYB zunt>NgkX(-Kz-Z3B{ut>*&CnSKn;xmXFT{0pUil<#kPv)Ery7IcTcu{Vr1Id(OXp$ z)(EN{C_1s&U80ypC&T94#35>0~O4TQ6N7@hwrexh0ppcHZvD1^@XCOI01 z-`(8}OH4HXFVxAy#@wC-)}O0sd)iftN{JTI7i~JMi7!qH^DxSfqq^$GuxVxW(5?-q zfpn_ThHUPxeCAajL%K;%Wxwt#M0twr4lF-g>~Qai@|%ju^9jFZ@1#jPS{@y!dYQ?H zMnPqwXct_5{*EtX#GthX2|;lJ+5iVi`g%~CCgq>slnU^R9fKhD=|yIid)7dm=v9Q$ost7rc4$SbI(VkF_;l(qEczTr50p1`}NVSn`|-* zDI}xjtr!g5FW zI@2H&%%gq&=9Y28`37;W1+nBsxAATMk29m)7tw%Ac z=kq2MvC@Eu5(%aWNr1@|n}z4VRzY~!3hfV+_7>t5izklxw2GUvng!p--phxn?}w)A z->_)0{Ge5seL#E>(uu-_4A;MyJVY^hhe^ZvQgi8Tkoghk1{LpE@ewtfX>`mey3PBT zH2MbpY95jDHt|+r6D?%6982A4vc(SBhS7;6sXQg=zVQd=sS06yc=609VkN4zcZQ$S zYWXtjr6#gs`*owaqnsY+HCYXjLq|o0RB*tV=#A<((iS>e_II!&a6@?(#r}iI z2lG{(%hfhc1{px)2I`P9iqX=_R#t^>Y&BwX8YlV2RRJK$?y4_bs`KN+2 zyf|xG3S;~T^l{)SlDb|{OosiGFsEQ#IdxN)w@};umRh{EW%4m4tGuS)^rwlCr$y~fjT+mV<5RZ)9~ zBJse%?74f~-148(ZbXc>b~sn&qXlr#>V*zs+5_=w#}FTTh#YtT%h zHEm(@Y50|j4@Ih;`|HQIy2z~e53susEkoPCx<0G=_{)$sotG9TR=-K6lNyrA!?|N3 z_<|iOp|~riV)^(XvzBT)>orFJ~qPfegdZpSEnq_3S2)4dOApwcU6IiWC#6B<%GeJ#^1pMbW z=WPeT+D|dNyHt#jOwJ8Ek)Dxj6d#@kh%1JX>z9B|&ttPBJF?t#+Y)!U!Q(QP*N771 zu+R<2!-ju}mEjQE`O7l$Vl#;{w(l)=Q9Z8?_2G3rI|w;!8!t$pD1&9V`o%iYawb_e@tw}4!S>U}xU58$wUCv}Pq+|I zZg|aOgUY%sx(BMPcyJ0vOfkTgA&a2&!g4zQe#)01Plvr2KtQhIK_jJO)k>#56Z=*r z$6BXa&^#v!PRTiNx{q0ZCK586r`;wEXfNruNSGGlI)@>M>6! zU)NC$e@#@8TkEQ-``UH?)w`7qqHu06_F@$>I9EZK&2X`hhCD@dixn6Z!0};lFa93E zLzJDz?HwIh6Q?q3PYGZ=JUk_14!}F0lw3DKKKRY)i7Rd@6M8Z?0acI(k^!fW4_p_= zT-VTz9aCX;hH^zf4&6AMXb+G?Ch^3Lc_za__izpV7AHhcj$)d=yFgNj@Z!-0u8>Ll z2F zSzHng2||@}Rc1TgM5ndKKe5LEckyS|)lumXKcT(1x`xlv!*M!Tx+; zINNo)zl_F^4jeWZ>=3k+t`C+QY%poljFFIaK@60snDSsT|Zd`6*Y$It)zCtUEUuQ3}Q2R)msB& zG~@;4=!w|+7Am}tasln)yUi$=av@w`9^`f6g9Eitg%R+`iDq+&?Us~0xEbZCM#19G z3+yz18WeU$-Ct+?3TuZ~1M=p>UzlTp=xff0YYbp)sij~Wz`7xv&afB{W*^f~IrLty z?Rv!s&y`)^jk-EGcZ}?>!MLL#{+)nFu}NO0JAR~QUcmd2Z#0%V>Y2f2=9p3*-~*w2 z?sNIKUjYCHIq>98!pKPg^QF5bsa~i2HZxbjCP`QMZFk0dyWGq*<-$s$5=vL;ZrsnXqs^EB3nF)k(_ z6rafRw0C4SY$CZ~QXpfZ-5Aka*>3<9gw;MK)3>jSY6E|2mvTZmJ_)t1Qn6}f~@C_h@q$*YA$yUflj@zrCXU{p-LaN6(p6~ zoBPjVn)N0(+>ke%|5$V+ouq`TeoQKg+zH%?nzwIQo_yFs@OSqZq&Blj8SYocarmIU z4AGWkHYDU2tx(Q;`7&GrjF+qWoN8D`UybB-U3jM=>4tQp@ebWBk@dX9_S#BXG4P1G zhd)X3NVZyGI^ArIb^blEpMo917VXt2%L|x4*D>QHoz(r0nARH`Xoqb}?nCUb=@C?| zaAG6uPWq=NjF>znjQ7Vj_Rbnh?Pb|&?!A%UK1_Ly)+AXVBpEPHM*SCuct#q@Kwqx& zYiXx~*E0T7LSkkMRMbO|hIhFh|6;NLXIeXB^o;$-<0q#0ej?8I^*FF~*y0wO;7QlB8hvv!TV!dbBOKsv#4;4Njw%A1V?BWv4aodo z&q~ap#6QnIcJj`p#!&=~%@#@OZ&m#=jQ;AFRHU0DWTmIf_eiUMvHH9;blSZNmU~5v zrlzNd#d|^P357mxplJb?L8dA$@!OJDj}>{a7jv{Bc;N@66L@b?&*lhB^IM|P?hZ9) z&@GQr2-vR{-TO|a9}gL@Q)O6+X|J_zWp{GTPbA?~jHG?sX$40bvzzhOTC6BrptQ%r zAn(s7)W^ror^4Iw-5$<)g)BqP$E^?I z!p!|1`1GxB;ez=c&mjmd!~j1593A)0y_ic2QfW`e=i~>a(M?Uc#~9QOuq5=ut7ii? z(3p`ocgCA)8*LC|JY+M22zDxcY@Hd(F%k#3tjDx(c{jcVJJ)OX(E=SO-cI#K{CP0; zaJ-#TlSCqotL>&}-q|C4dt5eQK3JV{F#*(hkTETC$~NpVj^T>ytMhL(RMDZIJe9dn zsD*Bvg*Y}1@+d~I-Ld^24135eu!XGd029Qs@t$S73)2Q{H)wlBqcHqCrKa{fTudAX z(%T5G^UG~_EtS_!cOkSjA@W{2bDSi z_zuRZmXQ>F6(tOhwiud=GOjim^fYSEofrK5%)&Z|JWjYQri6Zd0H#k)IB)YbYv1fN z1zL1t;w#=*q4aoAEH}bfIxT5yPT?z`F0?)nc{CtQsUq&13}$@lRgrrBob+TJmW!!? zRH%i6UI_FVnMOPG^UWL{qsPa`yy&f=8pyLL5})o+qxjcvdh3d;^NF<=U_(L*whDcq8NCc$b? z9WFqVyB?w>Dat2S?NV7ZkpG;1Q&Jaw&Z>8=*q?;FoHCPP>qY-MCbam_qJrUr>b*?v>{& zWf#5iiGC(_%Iwgnz4MD&YLp^ z{qy2b7MuQZX)g4DxAINuwGrSYE=;Y>hdDq?G1|9c6bPKP!-@3>f(zeDBD^A*^j` zyPmo0wWoWdFQI5|o8KN(k~#nR_?@MoJpR{aJqU#)0b39=sx2n;5lxVnrFQW9$n+xV zLo-`2L)!RgJfX;N8NQkdL5~R%d)f?r;w>0qG48mMHkp(73-94|Kosj|WYy<4m0|JD z_mUwVQDJ8mzVP>2p;WKLBlCGuF<$u2Ca5`;^m`pvvgR24%4_y)bUs;1&IeM2g}^F2 z%tzq7UNdR3S{Wp_jkoo{Tz|&Xe=K%;(CVa~Qp5%x#PS1s8p9b>`!760536;j`&ReQ ze)%na^@zZw`3uA3L9U|e$NR&!>WqjmI*+3J%*$4*0}LAYLQkHE(D$gh zYm?yWskdyq9SdqqwNBLisH3)Bq4m}t-X(r1M7SB&q12dF*M`TIT|PD|C3$+*WXXX zlYf4Mg8pe(o!Omt^EP5>##af9YtMScbECgUZ#X?xN;^teCZYA@|HH6x%+p9vs2OcC#2FSQfU-WFg5E? zJSF$f_#o5tlU=DT2;$`k`;cv$6tl@_w{LelL-0On=)RTj?a-4-u7HH}Ad4wG+-l$& zinH=1qGz7>+@rpsUV;E79?`BkX|lWvMyw+13)F(l?}qDUoPtk54y2Fn>V9*A`$rQN zCZUk&&^BskSxx6nzTiE@JX92w6Y&3~Uv-5c4E8fa`JA`&e7$m(WkY5IMVY39?_AEl~#}FRP+do%`s7^r^Cnhicaf{k&(NlkKN@{ zE8}*xBEII~t}I`bSKcMfCv*_iaLtakxnx4c zF|c8J32+$SD|z!xW2)Y5*S89|BaL-d&1G>(-7#q^Mddv8js5m0gYxr;vaC}50w>@8 zX({}Jf>c&}B~^ps22M0puAYb!M3};^RWD_L^K)%=dG5TlUO#qf{pFduyVKEgWqtF~ zV45+PWi^ny!hM?~*?ONyI668yeHS#W$pDEIV#NBZGxGwgKY4Y4M|;h0XAP30b>=B;;a zHLF$Yfb)eut{$($&(^@38yLcp@}v=4x@w)q0MU(e>4ek<=k)24gy?>gWBgBzN zH7F)Gb;}@kM#HmCTlgu71@e~zII9{aCwcOh5|Se6cZyl}N{2k$piLN~7t>xN&Vq{l z!}HGG%8%&cmv6Nl?obxOk-RdYo?S&^@JpCR-l~XSUhPhcz zv`ynPOCJNs;+NhrC}=ys1V720B%`&19Vrdq_S&k zSHu$CPOScB-{2=AP?RHdxJ9XWwjv5^<4~T%)f|Q?iB;|SucQU$(S|&tC=lC#52MhZ zKmy4brox3$9AW~{Zj!IC4V>l;VsmPXHYipQcP_!!Xi%-Q z?6+jy&!hhfb~8mG58+$10bPdsH&ad1xR4>ruj^bKjX}<=uSM=GiJ}8VnnfOyZ%;fM z%6CiW&CJd8@z^b|zkL3Y{$%QAj8F(bd`>shQLjU9hA9U^_JMkL_u30! zzTE+&3qnV;|98m{bHx4>gpet!@}qZIvL6T2ocGTL((Xb8*IuC_bQV_1Sfx#mrnA3O&IOsgp)X$e7_T0WSS$=^B=h;f|&_vS8uk-=ZsZ5 zFOXsrD2_1C6cdDq| zwrx(;sww4b-Upxn@yX*8nioingWtTkxnx@uw@U97qss+ zw$J+rMXAW-g17D8s?P+nyQkQ6M;xNW zZohn0JH(nRibZd|qpM7+w8eoUOGq?#w{DNR^7`z3;gVZJ?kT)<3(%mja3)Z3>)Q&7QmzctEIGGzVUQb+6jQke>z5 z0I9z)S5BSftS7a)?}|2g4C>t6T*-(SFlaHR0?JxH|D}6V#kSTJRbyp83 zxi3jXt8YfqeFOOgsKyDpPXM8;+Lc#C5);|qs2#HsIMAkn{?1;%7=Fi%%1kNlR$l`c zRSDS!&}uj!`WsIt0O&=~(2`QbO(I4AL3$9%9zUe}LV3Q+;U>Ix$f-=zQN z%zObgqR765nUu9GStD61O9%sNg?EU7U=TJgh({0p^|wO+ zxoYo20+p{Stz{|oiTP>u>w46!^p`p*J3a&rwB}rQ??#DTU8PB)0D@=5{~gdNL7sk~ z-(=9T%7)g5f%0-4stpx7+!&C2Ea^&-a!7fHa4tMy{Auics-*PvJFfki8LJ2#8MiGk zp#5G0z&Rug`p9SVeWtZ%h?^LJhFDVUvw0?*I?e>JAF9O;KlD2*)fmo-c16Nu1v9BY zcmxyy8G~|EyP&VjlDHW60zh}Srx4Qh!bAaaLV}XR^~KxiQ9WQ9xK(*O8xv{X+!f6Z z&;)#yLWM(Fie|u>J@@dF+z)@n&J4yN;H^d?2hDZ-nW2wyYNW{1K@-YJ!BfS~^J*Vz z0!PEmL=@>=7zZ8KDtp{+#N+R2lK?D_=qCi3S_P znqX|c{xuPE&`CNC7N4;M0<*70ULd$BBzc@ba4n?jvl%@DksY{Z{Nw_jVU|ItTqKf- zKNW;SW+XWR3ZX1wCjg3N8uJd$a1W%m{TijQXD2;|TbW zMr)A|x2g_b=!zq`)FG@|4GKW8;be)sag4PHA>$_=u`s&*Halu2ZJI=mzp|gm{gIqp~20OP-+~^DehPV zq#RVcOC?Yk84I0TI`Y&3Zz#<@hG=Menb7i{9(Q{i>mbes0uoS{yU3_q$$l{L#bp%z zSum!ljJ?9yw$)b_7&=^(=G}T){y-`jLy&k{n2aAX575W8-*xPU>Km4cl&dLP50ywJ zpwB$d(>2F)XqB#?AnyAo&yW3U3+G(gZQ2NLO^J zk44ChU4WR-dQiWBK%kI|uQzE8>>OqG%B{(sBIomy52jIhEAs&-GDDV}0TT{bj)a?A z8?MJrGWIrue&lz_ehtS2vdN&|0j2SFsod?X<%^ihT}gmUJKO!w1CKOHuiz)0Z8O$8 zO8h4ih(%1DVgw=G56Ac`=srrw6#WF!*C~0yz%(f7w6ZzxchE5P)|X*Jd+P_8Q`{-6 zU3s%=)9LHs1Odyn4;~gr2zpEMgq6D4WKT0>Qte-h_rhCJDFs+FsXQ6g0aYQD4*7Qb zHC5D&pLB0~uFN#YfLE<7`?{BE89UY*b6Lv~N1SVG;F-#Q%yo9CnmL(6T+(P&++Hv$ zV=DfoJxLK4Nk@Bkp>h2jh2Ah=@wdmQoyQa8%5a_ch*9+;zE(yvES2C>fUPON;6ciw z_-2)b?%35emksB*2l%yA8ZK- zA%VuL?hFedH+=6OZ!GQzFC7fR`0+^YaabIj8KSQ~8^7AYHo`Y+IV0}u!6upzz4%Un zYcXX}cy%*)!oWO>BT#bNyH(ACDH35NIKcc~mD=>VUL$p4Fc_HYiqIMgHECiVK0wnE z9JpJq;!4H_h%P8UoC5naP9d)V70`;uZ$C3kWKQgCHNdC9pFSe>)0HvtQm1D=q2hIU zflO}Mng9`+*BMw{MTxmN#LJtCZ}8ynVy8b~#@Dc_1CkbW@hM>3j;(abR{wk`E4pvdhnD+}d( zDD7*Iz*OH2G6zqCrmMZnC5Mah$jFx<>$H&o%RsxtY)naqB~k*-SNSzQRxy4=5TRr1 zi=f+Jcvp(vK2fATA`g?7&;0qVxdBHdzjyd&xdZ+OujSsjeo7;$iqmU~Y0E}Tw0BjyGT?6l`}XWE z#ABSC@@ADaWyj93+t`WHzr}wVBu%rCpl~aIOwt~s!4b*tX^aLE*}VX!CNC> z=4OFmY#c`{PZv>&!d*H%p0sZ?RGjc$Dx+7`7#qNGZs=`9>s|IuI@dj`Z)0h}OKqQI z-?()*({XKZRO?*p3(u( zrK>`uRtgVCUONHmwa~6TJo5`LDjc;=gIvTOXb|XKHwii;QCuVXV9a0qS<>*8r$8o9 zA}FbvaTUNz#~<6?x%?1%ICB%ENQa|knYF3HNT{AbP;+U>`To)HU5H@Yl{pi*)Z7uZ zeJ#S_V{&T&!3$Q_uc&1UJ!UVz%MaBEUB+W7zVE6k5i)yX6+!JZGGE|24MbBP+F;2S z^j?4hbE9y+jJPU5GuTcLCV+fk2|0YOen#XNeLGByAPF)n%rWzx(*%^~cq=^cU54u< zXttOD7W|CyW6tTu^^bjA$TkT|E6$*%rY-UU8%A)+s`AJW-~YE72U%UDh&IC6EFjkF z%s&?gZg;-n;}-Paw{-=$J%snQG%ebU4t;hd+sdI;JI*m0dzALZPrQiifPWl6)IXZU zg&u6Jiy z-ZP{k(zXRt3Eu;lEp}NMz_<}l7mg=Q+2AEpfpu56#@FLzU3Trb?VTg#+0A{_xX`f8 z)#-s6=w^SO&h86{GP;YCaWA!~zOs!|X{>$I_u>im|;z+Dw!?LOyWH^z_s+lC!bRv3zYYoX4OU<12Eda5bpn)n*Z{dZQuSa`iq!wNm<6fD%R8!bYB{lpBN^bt` zAPm-Bv=O9Ha+N$gIYr4&fOS1*UQ-Ku@mg1R89RXH+!3wK1`fbW1M!Fj00>Q{=9{X4 z<|n(-9Y>y?s58FtJSF6BQDFDq?ybA%kXAu|y znU=zvSJg*BN7+TtlC~YR$GWdU05133^{YMD!S=4MSJEMunPAY{pvT*39Q+v)89sde z8E;U=)~^!p+oXhP;L=&KrBSjcmUF~qFAZ!k2K&hC(g{cS9M0Gy&aC~&eP++VUzc&p z8(K!l%mV$9Ge?b9*N;f-vOyVt>T|q-QZFmbYDyNJ-hZsWTn=bI^HEFeErTzu(A!UZ zf&qB!{BT>!j)0<~kRa12IYGL*mqAZHP1hreiMSbsN^W4H%vLMinRW0~5Bhuxo7xI8 zlYDN`wbC>o00%8$le}&Scs=(65U;M$<;psTGVGN%0*8noQpcE_DqSo1h#E zqk$)2B3ROkLTP(Ch6VD2s}p3^uIrtWCeL!v)_jU-3x6J-WD(HfAp`&%dsB{-+1Vf? zlC^V82rz6|LS0gFG74dkr{sA^OzN`v2?C_Z>hAT%no5@?IYS}x5J_?m&ARFG{dplK zg9);pRv>*1F9ZVEA4#AxH8RIVUjLgR=OF6&OyhW8iG0IqHh@j+&~~WcY}YXDad@_> zKHf8YkaD9?()6F< zHe!YQJP#q%Nil6v9QQ=HiIu&-=o|1h5gY!63(~*@0#)>CU{mCp@5^(zENV)|n!dzz za`@nHmgY~zc;M?-0jE#V33!?^UlT_ailuD5t_F@YI!qk5shbaXw^|2mostWf#j+p; zd(c+n!N)YMj;cQ>ivN336yW2fKB13GvoMX5rbmB1Nem~ z-^dHU2z81e#pSSi^0hOqoi;GJm{x9=5@uv89M7fNzC-O+^q7eSX4 zd}ix@j%X^K1VP6M?C+1-OgFsky_w&W!DHWC=RjqQIh2R1rgB1L8gBRBoQ`ka36`oR?GTRw7Kpokvdw+(jC`=XL zNJ-PgZBg!Y37pNkuCKv(`wrwF!V!;OT4S;8x{nW1jq^eg$$k>i0nV*=k`$$Sw-+Y{ zVul@CO6knYyZL>SsVxEcjW2?x2R*jW!y-h*9HDtI`t5Wy4GMUI#*gX4k?;3+1P`B| zzVX;Sfy#r=f7`Vx_8MwRJ8NQZ{BmH05S6(2{z}&0ro|qL(pSIjw{lux6}5rUxzo`* z!_Nq3*7WlB-X%>7*XE{<;tw!3G~cK|w>0;Bn70*nouQBJ?XEgEj^254t!&^JP^^r` z$k>EK4jQRJ8mzZ3QHB*f$dhBLyxaG2M$my>%hLr=Vt^@EmRuG)`C5Ay^~8}AmC1rS z8x@a(w~Aa{?D~opTPCFohKNf+th<;catTK_HB(#z{X5qPAJ~GmD6*k2j!F(=KxCYn z+7{N@C>#qj_Ne?Fu0)x^ecQi1&N_JheZjc`P*Xpyz@a0Rj+++yfPQEI5!P&0?qkmo zD5*Isl^9AY*m-)U=Fo&yT)1nh&HkcBQp3N`bT`yM8%jNxy6=FCch5a*hb}i1W#y&| zCIB2w<8k^?S>JyG@1YO~>JkLYbKLMxh_D?^9u5TvK&Q6%`N5ODA+U<3;qA&pBp%*n zS7q__oKHLl^RS`cl^2B$om zaH>#|J)dZ9hN;wWj{ds^`+a?xjzdF9_UytVvB2fuk=9w0?4uA6x_!T$5a9!?2D~qK z7wka<{w_y$)|)9@I|AlZ26KWrb*gjU+AH!c+Y@(C^ton=f_sMf#c{oYH)6~2f4&(` z2^j5Yq%7fP+IGFh?^pQ!6NDf{_+%Vk-mGKQ&ZGSPuiszfz)hiVFaG}dFQ1r-gX8w4 zJV@cx`5zhzg<+29{mFdzqfL>3DFVk893Hhg&bG6Z{(KzeL^=p{n*TYF!XzMS%mZx$ z{}oBU4B)4?Y}gfs|Nd(`my>J&6pF#r*(seL#Do7bDsThkemec1FZKU%14&!dJby1y UCx1RuO9lS)b&R#kG#&8&0fFvfGXMYp literal 0 HcmV?d00001 diff --git a/docs/trulens_eval/guardrails/index.md b/docs/trulens_eval/guardrails/index.md new file mode 100644 index 000000000..563f3200d --- /dev/null +++ b/docs/trulens_eval/guardrails/index.md @@ -0,0 +1,93 @@ +# Guardrails + +Guardrails play a crucial role in ensuring that only high quality output is produced by LLM apps. By setting guardrail thresholds based on feedback functions, we can directly leverage the same trusted evaluation metrics used for observability, *at inference time*. + +## Typical guardrail usage + +Typical guardrails *only* allow decisions based on the output, and have no impact on the intermediate steps of an LLM application. + +![Standard Guardrails Flow](simple_guardrail_flow.png) + +## _TruLens_ guardrails for internal steps + +While it is commonly discussed to use guardrails for blocking unsafe or inappropriate output from reaching the end user, _TruLens_ guardrails can also be leveraged to improve the internal processing of LLM apps. + +If we consider a RAG, context filter guardrails can be used to evaluate the *context relevance* of each context chunk, and only pass relevant chunks to the LLM for generation. Doing so reduces the chance of hallucination and reduces token usage. + +![Context Filtering with Guardrails](guardrail_context_filtering.png) + +## Using _TruLens_ guardrails + +_TruLens_ context filter guardrails are easy to add to your app built with custom python, _Langchain_, or _Llama-Index_. + +!!! example "Using context filter guardrails" + + === "python" + + ```python + from trulens_eval.guardrails.base import context_filter + + feedback = ( + Feedback(provider.context_relevance) + .on_input() + .on(Select.RecordCalls.retrieve.rets) + ) + + class RAG_from_scratch: + @context_filter(feedback, 0.5) + def retrieve(query: str) -> list: + results = vector_store.query( + query_texts=query, + n_results=3 + ) + return [doc for sublist in results['documents'] for doc in sublist] + ... + ``` + + === "with _Langchain_" + + ```python + from trulens_eval.guardrails.langchain import WithFeedbackFilterDocuments + + feedback = ( + Feedback(provider.context_relevance) + .on_input() + .on(Select.RecordCalls.retrieve.rets) + ) + + filtered_retriever = WithFeedbackFilterDocuments.of_retriever( + retriever=retriever, + feedback=feedback + threshold=0.5 + ) + + rag_chain = ( + {"context": filtered_retriever + | format_docs, "question": RunnablePassthrough()} + | prompt + | llm + | StrOutputParser() + ) + ``` + + === "with _Llama-Index_" + + ```python + from trulens_eval.guardrails.llama import WithFeedbackFilterNodes + + feedback = ( + Feedback(provider.context_relevance) + .on_input() + .on(Select.RecordCalls.retrieve.rets) + ) + + filtered_query_engine = WithFeedbackFilterNodes(query_engine, + feedback=feedback, + threshold=0.5) + ``` + +!!! warning + + Feedback function used as a guardrail must only return a float score, and cannot also return reasons. + +TruLens has native python and framework-specific tooling for implementing guardrails. Read more about the availble guardrails in [native python](../api/guardrails/index), [Langchain](../api/guardrails/langchain) and [Llama-Index](../api/guardrails/llama). \ No newline at end of file diff --git a/docs/trulens_eval/guardrails/simple_guardrail_flow.png b/docs/trulens_eval/guardrails/simple_guardrail_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..6b433d18b3b47c040f5a5ab8a77bf742cb5dfa43 GIT binary patch literal 31962 zcmeFZWmsI>wl-LWy99!}LvVL@3GNQT-3jjQ8r)rjTOlDxAV6@p1PNZaLs#}b=ic*t z-M!D>{?RidK0qgNj6e1ONa~Wcz^G(U})2dz{#}6hQ5l5Rc`HQ!SfW6=^35nR65KfXI?sY?7Mb~ zDr=a3M}eZ#t_f^+c{aXrPB0L zlFY^G<=;i~<{yFM(U`N|J|Drk__vraS;$3J$s~s-K;aoIm9o#Muquf>h@@Vz+oaKd zR*F`_qf^t8jv<1ZU1dX$ZF^7gzLrcZ54-Q>d9Y1;Iw*^W%z5=@;i%sN0k5i6=pF&D z@M@lZ*8XI_S&iejy&}D^jIHPy`F_K=C@)+K32$J7%LaeZb7H^JC6lIH_fvUBNU6X@8x0+I9VJD6b0-I8(+^H&7R=rbf2sxm1ikqo zrw$hGrexj@_Kt4+-a-_A-N6qz|MN8q1=(M>xZ4R)=qRa>NjSM$ka07!GP6<$BaxAj z3A%o;`=2rU#9f%f|e5U%#N`MaZx@wLYI3<;ua%?iN2BIoE~_+H0&5ZKCW*`(Q>#IxM6Kp zKV5swnVvkZYWFLrzky3^BSo$k48GxAdj|UzUClpxvzZJ(yElMf+$w&t+ z7DhJcpAKu7Eg?}Ts((C8<%6VySV4_MIRB+>KM?wT|FJemTfU?L;sK^UrhJV5s1V?r zLd>Asf7Ap-Phs>x0^pacFRnWO+yg36O2FAaj{xuuf;vRCVF%&6jsL7N*&u1a2K0Za z5ThK%8U|q&dEE5Bjc7ce2mZfQNT!JR0|BEPH)D|EzjT5C=#Bo*^CwC46GXLSiY$zg zQvYS#Arp<{|Ci^+VqKF=m zKx#N4w0%$$+uvold~tY;f3a*;@a-5YFP0z=#c__1%#1-JU-| zO!*Sr-vx5L2@(q$4zle5X!_aKJZ&}ylyG*r>k=6D7z_z*9gK;a1j_sAdhYi^W)Y_3 zR(N!(T`mQA2v~bu#W;&gBK?)4W_L}H{_n!95$0OaS}G4gx(ikPhZnh=j9 zbej&MRR%?qqh6u%crTpE+ZIjr{7pspolYtPXRJbCzfxEMq#d$_H<-V&h>}B%HOt~*lAZR>%SojmZvo7Lnk2?c)#)TA-^9F@auFz`&G|oAEnI{2M#Sm z1LoA~trRA&J6hHsr2CQphy!{Fdkd){Y@StKiz6ySsgQlmrAM0@aMAd)YB7RgOb~*^ z0Ar?pBkEsL0rgDkfbjj+-AafsMJfOVDCCHTXo^tjScF;F)gX#0BvK5Z`y(g{wJk;V z*&~I_0gt)|kqjy5{&nz?z|H+nr)0K*gF`0@7teZ)7`ZPHoWK4pK&A4C@O0*8%$65H z-q%G5vWBVjzeV>|{MX9cr7@X>uKOfo?$k-XumxT7IMf7JV!8-vh-{%2mM9eEh1jdt{)A^+T5ypd~AkZr+Y2=BP{ZG4)EZG|b2*M$1 zXPHy|Bf>?=LNHo#$6~+mk1GD(kN>}U|4(X!Xz!ZBb&kk==}pIOhJLA7NyLm`S2(B5 zf(8MXBh@v*TqeKA;b^Pf;?7vvxa;;4ET31o&ievE>P%(>6nz{byniA9e|CQ|Mg-CO zf+ZTMuB#K~ZwFN~r61L8Du;s}Pzhw$eLP@SMIMhiI(>Gl4SL<>n5!>dpO($Otron~ zC2Si@4*h645FGli9VghCI>_qddY8xUSo*tde#wM?GnwizKz@~ndv(Q>X0m?uPOhd~ z%G6Hw%W0qMb@lzTG1vZ%?w^TK=!sE?S?Fm%d(#91#bj3e;jVO*s>IE=?@J)fyY*ocNUF%Qg_@a})6Saqf-$me?J8*M?F zSSZ}Lah5*D9Bzl|4Yc#)f3I`}M9$hxQMF&cs42-+33%+SiS}#{RQ$T@;DTZg8C6u= z#cy^=lV|rQr}4HFqV4`HqCZ>+(`S3obgA7srl8*6cl6WaFk;#RbxbeH9S6#sW|?^E z-gW87R&UCRuwjPZRrK*jiiH|{`Vt9G%D>5x20suFoSKrd1KHTpU!EUxF8y!2H`94G zyd`j0j2Lm54JxmM!7%@7CI&N%9;XdOsDM{`cO*)>>AKG+>mX!K>oaZ6qX2*U&JdCvk_Al-0c8+pq%%?6H%b**%Zx?HKa&K09!h2esd$IF` z2$>~zc>kIpYY>J;$HPyeV+fON+xz-b5+Qr0|LBVkD12fOcGE&a+f_K-AmVGeEhJ*$ zWx#F4V1+??m9`a(Qi#i3aF{KP_jwtirInq z)eXi2x0=eo^Qq{Nt)d+-KHYDd!vsMa%Nr;hWKCTR|Qpu2Ar7KZDZ0{2Pig-`4bWOTOf zf4bQc@tTy^*>THvJLV~}ZO?hyRR5MyrDBT+5o<@~k`Kf~&7FqQz82h{`{7&+4#VCf zPJUV&R4Y`Up2k99tHty}5D@!o5LE~f`)B(!YqV(gUxjYhC{OjZR3&D3XZGMqI6UpT_{Pk<~ObzV5p^^k6Po{k0VZ%aa3k zc@F_(uowZjq`Z1`4U_rw$lUb}X_uN-C3$)kW0|z;OeH=y8J^K{GwuE|x-B154SM{l z_`8SP{!ByBRxAi|tgrwDFc^dR^`HS`4;vDX3aOS#LXqV2%6k{BO&%WQMCwz4{x#?J z?`(h(Axt&gqxa22H2j6Sv3vRJgx0}S)356xKfdfut6It*eKL(higZK!)=S*|VXDoZ z&+Ve+5hLJV8!p7~5H0+8xyK&xhn|r!P2~ANxyfQ2M~o^c{oNtx z?q=0#yjHh)r}y!c1FOyBU`8qc97;N>i+UePZnF11t0~~wCzw1~{9lVBS+TJGCBt#1 zFHWdd=aSNj3goya zt-|BY4wn)}{WisgI(^(mFiQ`fa)B_DdbwgUpWEK(Xfmb6Qln|P5FU=t)o;^s4y(!0 zd?DZOEl!)l=0u}YQx&HDQ2V)mQ35gw1hTGm>5gCRhx2v=e3SE9yo{O^r+lPEN`~jX`2I_UlDk!9wrNT4t%w0b;vm^zq5`# zH}#8u?e{$c(Vq3a_PQU=$({DS2DrV)gJ!)xm>p4Un?Tmvq!kd*_dH#r@Af*&=;;yc ztekor&tyR^M%gk6vXI8mG5cRU{hPwgc|rzt?&cjH{j)YKblMHKlSqXBj%fY|=6Cuq zqOnWXV!f}-{_~Hg@ioWdec+vxE>NylCzvenSB zGMKCTT@RQKpC6Wj+z{0}1ZuJ9~yS*AzpE8GLdmUEWE|$y_ z@rQ(BUW-Wpf)CHpLM)T(~U2i}iVO5_4T1cF~JJqdu9LnZ*r2RmHICin#;A z|IZ$O35Sr0ZCC0+Ya*4EK?M;#x7YRs@qQcPRe5>o>P3W0^gSD@{-P9#>v-|r=ob0+ zX~BojiJogD(cl*WL-#T4oAnO&_-H6FzbYc>WFCEn)a?HF#>q0f-%lMmSta%=3}*y2 z6eXXZ0UvMY+jST_U?SEe|3VrJBbbF+UCK{r#Qaq7=p<=w`;(g1K`P&s^96NDKJH*n zaok)My!NblEE-gU6?Y7%SfAq96dI!@?kB zZmqrB{bW;tfYW{p#L|PMoR=Zti$1o%mE~%(*n3fSI_-&eI+?7zQBW5Vv|6e(f|8g* z!OUjlNX#RT6TqrnZ|!J`^e?&_MhO(mtRowmqAl<<6Up#RXc0jdzAHq*XaDMb{L2tv z6Pn~h$G|Y=4SxNJ+sDfHtcr(s4U;s|C+K>-#5Mo}Ew;Y^pG2vf#GNTH49NgBYx0y~q!Y58ARLFt#xWd4D&aAmba`Gev>hc*e4JPeyqUvSUG#65xF(#_-VM|~ z;A-h>nV|!;-@bB9JQ^3uPn{g`i*I`?{$!RtYy{rF71l5`Z0i-zhU!=@m=WW?7wZ*yH8x z_{JHl!*w^tDqXYKS1YY42wSjy-7_Z+pJSNrr52MolDy)z;PuH$x9Nji)wB;Qncx$6 z@ZyIAdKO2qC@TtljdlaGx;(`kPZ00O2Q4ru3)J0vPosAY@lPtIP_k~jfN zP{$5cxH0l29GMz|^hu7$L!IjCAa&oB0i_)7iJD&cmk8CGP{=;7!hyi)xTIp1&7CSu z@Y#6XRWr~HH&2MN;@v0P#;%cp%fASX$Daw-HG0`Y0`<90Z|ns9eahU8&G6Awvl#AH zSz?PwT?&?DphLD&CW}#u0)J;2gvGG$OeP;bH-rZ|9oyaV8T-BGW-_J~;Bnv?3RB(U zNt#oK743E;=AtO-{>))-*gTDg3Jz_j8bHq%()W^=-uDba2;x6fLP#9-=8~mqmCFeA zb(QBAEN%EHA^hSEF5Ti@7gd-rR}(}rd5~>}h)J9_O4F!> z?CCVuXFA%i8d|1M0KSTfJFfXYy{H8>$tkE6eM+R@>e>Y14+`SqA#$zIWn?oCCL28; z-KOfjxeGJ;t2Jw=I5^aAj+cLx5nUbYdL5#}f zrw$15gM~%bC{eAD-(6#_;72th?`_z#yu0W>?i&p2b#^0~D^e;4J5f+>^ZwL$59-l} zWO~or=#Cln8l}E%{sJWYd^!@S#F6A@E*k`3LVTjSCvj zh_P?YPl@-<0sVf!B2ohd&(>!u<;>&hz}x^6bB%n*uYb2~BA ztyat;m{>>c>=LD*jxWehf5zfj@Sd8)Ex=VA@$fhmu3K*OOl^sZP61NF$sM&h^my%6 zM3Fn|oDsf@ydf>d?EkWr3u6lij>UB<-*1+8w`wQx-U8wac-Fe5pC60(-qh~w?GeB} zXE5o1I9X|>fknz+FM5+>iI5ESzO!`tw5k+4geu587eX}krl>=SMK>D6v+*rI=6VZ_Pn786-I!?ml`oC|aNFClQTjahrI`iSRwS4b z9$g^;XUDMwt|{$mp$v^qOdYUmH-|t)pMDS|4?9dOTZn`xT7*`{R}6T<2iA%c%LkV3 z2WPTg1P#^@t+Ixzos)k50bK@dzi`|IBt;sR#TKkdmbSUS*xl`-iNfJ+$j^qXydb(< zF1rP9vKEKc3^)KfDsz(P35E0a1vNddE)_fyay95zMNI5&9RIqiEeWtk+oCFXZ-~A@ zp&%h|w@>(1BVLwV^HTP+43!E9Vq{kuy@CpN^@w}{-%NH7Zq+_tao%Y&MTS#7zm z@RnAc#k)x7+#5se07I`prJ@!VyLQPTNd*o2mi6x+@i3Okq&iJ8Y;ooV=_o}LgF#m$ zQEC###5S|cOehG>2viW7lSBbWA3(t4c+9~{NDnyM+I)Sv6<)0|4;MM}6ufkx5~X5j zGEKXO*sG4k98ZQ=cpr;PN+w_eq@<*1q>@$(**}b81V*|U!eIV}m_Ca6g3IbyCO-D_78j!zhC?rJ|k|Is?l!H>mwD989u*`n%{>FmDTMzHuZAOb6>zBNNZ8| zR#Sd<4dO>u5rZ&$4Gc1L7E&PHoWS+zVc8O>G{(&P)v6lUZ`@VTlzWitgP4|f+xK}d z$0ABq9{2EB>?GD*AT;8T_(ua*Tb)E8gbHX z5MUDjIvP$Y7jm}iwefMJ0az#ziN@{?HaS@9Y>rH{fQAOxGLbcz&z36$Ur4XM zxEKk+fJxxZF;KiTJMpIp8_e>--wtaU;jbmpn(ZPl-lV!x8dRA4J# zwEBR`Xv5=334Bv&_D6ec&iU_~Zz?H>$F3K^d-IV|>7dnSi(gG#Z@B?WH%}AW4kQJ+ zbj0&Hea$#84#C(QEAZYeZ*8lS*Y%sJ1>ACS1&uyn-+~`xl8twYotEw_Nz_a44(bj< z+4*j_r6u3aX;0nCrp-!Kw2)n*P5tP;I;DY@M!S%d z^X5)#RB0#atCsIqKjJ2*7%rWw_7Jj)ggCsRFnL{m`sCpT#sqkOlGu_mQiq|BTML0| z*GlT5OM$t6x`p(55q|3R@w8jn=k}Bs12&F^4oT4RJ08b!WnvJ#GxWb5xT1AbVDKk_ zo$1fNcB=USxmp$DSQW}+)hGHE11p-Y^6^rRS*@=0@c>7KKB-1e_?fW{+nzQ`eder9R@>jN5MA9ps`AO`{px~gVw`*fk~{8CX= zpf&mH-E4xl zxn^~HlOD-UJ?~2d?5hFlTkUahK~1h-t}o}_s-VNprHLl*`H-_y6+X|>D$9K?uMaUr zIN`DJ1SNea?9HF?{kf)F=T`V^gGp@h5b9f89AcfRE=9Zh^FH>?EPMG}fX0>IjW=F* zQWx@YJ;d$pgQt(K*KJ&Q|8fip=GGj080)2VHXMMauRWZrGmnccTgOUgSK*e2nwKpE zOM3@g?)73CjbfokM@OG~Cuy2%PgFhcK6w$)+ZWVy7ihY?goHrpb~qO;R)N3!op$b1 zadFMt`P+~_&4)RYxIyfK2JyiT<0q*^-b67f#2F>s&k^vCd_UI88O76(Xt3F&yBO9Y zI~E|ti-8Eq7`_rTJ>?Ouq7(CfD1Z0h7Z)G@$JYygn4z2a_>P4|8I4rfei&Us9zuy| ziQl5I4j@cM%A&lB8g}VDBl`)vFKTLp*M@PzZqgt0=JGN$)gpp#L%SbE9m1V9(~NJP zJWoGFctq{lEjN{|8*KBfz>7G4{jkw67X>dta@UObhd{(gqyXgqbgkFQqC+ zCiZSp{Z~hokDHam;ro->wGK@@$_1}_EG0&uu5wma8Kwjj8WRmIKAj|%qo%g;&+1a; zQ`%{AyPlFO5#E>IToE1X@m5HcYS)ne9z8c7yDQD*b<-o5cOrAys`Z!2sl=QvMCtkz zQv9pQbrpwR(8WMESFL1L6NFw&Wwc88qpZ6{>AlN!2{0-4G!!%7ehB$qVg6L!dnA#S)fVN{N!wAe;N_VQAOdC#P>}C_GHyO=cY$Dl z8oPFVW~eQ;EAkBC_1pto2YRijYOlwM<#MxmCcV4;qIm-8phI^{X22kPcJbV#!=%Mr zruS`YV78_Z{B=ft=)3I0PY*wqCep{!s)qBHRq~Y@KZ;vc4tK%ojzh8_-n^VmvKW-Xf z*&lPX^tsRW?a!1pokd^jeX`Ga0sZQ29+p_G=z5@tx{~teGOKh^AMr|94|aQlzyT$b zV8F6=qEWg^zCs7|yA?x3hBfkt?GqOLW<#A42O#9O2jd-`+>Im<>svwx3AL5p=Q#D! zv%=_P@wybF5ORMHxDwcGvWQiLcx0MUV>ZtG_672%PGRXlY&4~`7hZWMFCE%W-6z}! ztL+WS*_7Srds$go>~;%`%}7Y+xW<0dbSp(BqA)|nQn5XEV#~y%2mp(xxhi!K_U(;< z$`d-Nuz_>}k#2Fy#%bdHbcqb;&|dKQYBuCbDGA~o`W8tV@Ou!DOdLx zxMoeWe?7om!KorO`J+g};%gM9umkU{8~dOS9^cFJ=IeT=LT83)qxkgUL=I0*$j@Mc zVDdt%sr;E5Z5H!m#1_)Uzr!sWxuQ8?lK4M5a~P+uL>&3bW40I0>(*+73NSUnr^T@B zUe`!{(=eVR5wn|_Y|n6`slRA5tu;h_XEPX6KH6zjLfw}Nt(_6^hZvpkk2-Ui#T~CJ zA0GHtr>|GbzOMVeHjz#|WkuhP?wC@-4##T?`)PZUc2({7qbcwhzACMP9DMI@%$4Pl zxq9F!k31bSA*ltKLXob}r!R@H&G9bVZtaPg0k%DKayU)~v~Fn?Z}IFl?j{qzkF<+q z(URJPMXZ{b-{!GJfhn*_@jLc3Gk8qC1C>+$YB~yXUoDecE6sx0B-Z?24zPO-Ipz z%m!5`m5?viKV;IEqfo%5RnwkHm!vU<*m%9QI8K5Zwf*LgDL!^G*~d#A^Bf=ODrT>I z!t}Pd8hy5rmP_cfpb!Hp_m5Yc%9#mJ*+tWLs6YN3Y)bTGBd-QjWU~wkybTEKh%0EsWHKX>cO%EpFX|w6x20oe+--MHy!3MsIwPNBN$x5#e z!~j8+AXDf?#1Z4n5IwNQ)M}dJFWq6y%?3@IpCt&f@13qv*Qd9SkfOj7X4GBt%{6&C zITws{7Wn2o>afA%Hi2dE8@sBo**?_!Wh-D9bdlNjB!xzDr3Ek%vC^aWCEBxFs7+23ei*$p@Nyhv z@3}EmSpxaCD`sY?bo~YUmh=~j|$&^8RBUi**&Fdhju?Jydfp51jPfggyyk{H7_i+dzPM{~d`}`XYCbn2~6y)4e!oCa?VuvDNixQq3(j;5^m0T>#SNP;*xM6 zy?^AxzTC@pwK#^xinrnxd_6%IpgV1=o#fnD1pgl4#-moDB$Fz^YOB{`q0`h5KR=>; zPzg@qg4hGtJ0#rU!6Rz|MrQg$A3dc&&@C0&YAm6Hu&5R?hiS5J1jCTgalIqfihNCK z8mubVnYV~Ym2<4pHg0}k*YN;DCG7ob32Y9ln;@=)N{XA&0l{Vktv%1wE{pFwt0&~t zlU_D4wBCKc48T{B{-$}ew6F`utORwWM+s^0SnZUyQ1r3gxxh3Er&{n^HYfk=1ACfE~3Gt3| z!&ajoa<6r>@()-Cf0R+O&{SQnhHd1j9_dZzJ}gSiCXp2jb0gJi`}l~S<s?vdsIL81bD+dKlBp8>R z@-?d%W?aSj%Pc;US`L3DKm2b+7V}4-lLf*sCcQLZ4%k@b$O!KpBS4IQ{IJ3b@w9ZG zue1+UPFwXrOWWx1qiA%TcC>kG=`1%}CD!V-%Kz~IcUOPq>zU4E(%)MI1VKNSeGMrZ z!k^d@1U~E^qZh+kPUar0QUA8MLPhckgadH?WDIa1l!8(0_B@F@AEx)By8Q9rgl9K#^Ukxs%@pvd%{^I&s*-O5S6HX%$I~QAY3c3a59N)A z8acKdr<^_y4lTxMglJ)!hknrWqWUXl3$Py&E*cojB^4UC^R!PD31FSSsfB-EI{*&^VuNlAT_IPu$H{zjt*{*fa&xy)^@kL`Aycuo`Vktd=>%OebYF!D$g)fO#x!*x=ba+=}y)riy zg(B(;-*o#eXu4f~6BAlV_=cZ5Ofk?~Cpa9p^UD;!TH%tzwjc|CwaBmr(00gw{p2C$ zd%GP2jyFgm$r5Vg4B|rv+ccR^O*T308|iX;eO`u>@@Xq~J}4b7nE5cCeMFKH>O^?h zEHVRfKU~(_d!J`>sN1g|csi%MS42ceN=r!_&&CDF-4D7D){IAtwG6P=qO?FQR(zu% zrygbvttiXIb@18OZ|1phJwNoZ((6GGG}6!)<}kx4aN-W4H5*z^Cdo{e zoy^6XDZ^O}ALyj5F~DY9=HAXe}c@a zX}%7%)TD)UVVSHm&!R=YIiXDaRGunIOYto1mGDwH_y=y_=_ScV&rUVGS5CHhr>3TE z@o{D<=d{GS2vfs@Ab{m z)J;gR4vRR*cO73fgd0rqyS_8%T2?dE?WfMuGg;ZfN~<00+2yF0G0C^d$(xwtplK#1 zriq=45soz*d*`;qQFOd8ey>w|HT<%Q3g5CvF(Dw`v)A#`HXq7!n3wQMgMDz(ukhaBb@`(&{@crd}&OXnn3=*OwMX(eO+UfBA_DzKGh^rxRuRhO6X%Op)6nSrS7VGRr zuPfiv zh6dv2HQ{b`MDAtW2z>SbbpRtJ4oM{O3y9LVEp$xQpL^U8K&Kb9YA}3cyfU0+@$lA@ zaOrZKMp~{pFv}GQC=?7ZzO0i-UeuoL_)6Wv8E%x!|GS<=kgl8iED80n3^kk~4&em` zJ%kH6(+(-kaK>2u`-e)`mS~@7f%Fhoynbk+7#%tpN~y_14-fg+-#-G_1|Lb}9p4QO z$EUe7{UDNE@pQC8fdd~$)*5#$9B($N4vI#4FHrLMlZ67YS37Rr;WBbP{tAxnlY+qP zD%pyJ&nF22O(?$dTWcF>fbTIYmj)Ml{~MdSVIf{Q;oOUSdPx*%-R$8C-`9(G?pk)) zjQZw9Ic>iz0F8Mwe8nWvHL9o)fVF)YPCd9l->9)lhN>}NsNwYB5~x1bVvLXIby%kA zWV`$K3@g)Qi8K#U2)osJ%dw~I-*4HYhe{b)WM zhd4U0EIb*@k9?{j=1E$5?(@4Sa!0%GAzgb10R>mn6vny2@Nl{$@;<-id64MLDC!Q6 z{XmJ{BTlOmlU*OH2Y+nv3UeYy68q)oe+g7ECIVfwqLuX$KP(r}VEcjHy)(EWYuuThsO&2c`! z*p~R%hbMGyuRe!TeQAOF73f6yVxJ-jaL;`SLldqIb3l7gya;8#{-xE&gYsh+aPRA( zKf2v|=n;R6A5(&W+x?p?kZsob(D$Onht*vCi%7C~z|9s~mBUV7f%gKo$1p!6AHoOE zV(ll(I=b4MB9qVh`Pt6x#)*E%iJX##vrn(}Syho z`>p97LPM%$xHJ`R%1_ESM9HXy({cp3?b(Kmd{GXWi06FU%Y0}gik8cq(B-t1&n6(L zQI6LHk;MxK%Z`uHEbnveiptAuYDoG)t=2X%4BmCOT!QVZHtu4OWV>&wPVPN)(jmNA zCH>5{G+}B<5hE}Cp=yG45(gVRDl0gZ>^PkdMU~PIxoAEP@^4DL392{FsO9{M+lvM2 z%p;Uxf*$YY+H6QD`?JW@rCJwz`D1sxR=-*`Mhrkn#gT*KSQ;<^)9=E0(iUX}=G+P! za6|dN#RnFfm2erVWgkzhm@kl#253T3s}J-0x4I5*C2T@uG3{VPIWfXvx-f{z}jI=WF|+x_H&x z1$w5EV(6PGh*{VvHz<7(A1O>51=Sd`j~WNXyJ$24I>W~9mXd^~z^ui_CeAz#g+G>= z>gguH?Y#zL8J)qiataC@kSu%r4mzF``n`;c49`^z+z-w4XANQNyZeqD!lpCeJ7TV9 z@7-#IC8@)^8zzlz@PlqsI^nk3$K$yssTqB5l#Z$XIH27oGv$z{EJ(5@v&$$np1F{C?H@>m_`fb+?>1VrgP$MbK1TyVn}f zlfS-r;78!S8%-y%4GvkB&)hKdC*k$XaYGx7o7G7wImneywbc7nROGXiNu`>7*7trr z$GCz%o3fXat(DU8IIIhHK(=wnLg3~%6|0db2I?=)Q5ZLp?x_JGK2OyYwN#-9rYU(5 z(7D*(qND>wO2mm={WuD8!C-^h5Hjy?`@RsDHH{MOM8U)iFVCa-w z`W)B!#SUXEMe)pH~cP@8Y;tLTBgk93MpWoga(rxQQ-Rfo==?E9acD(+Z=M_ zvb2*gA;~M1j!-I3XCkgTNqgl~(4zLSboV#=`BybRM)0&8yKNNP%wyDHJ&wGcM)z=jMj}kBzam9=I3ugtgI1GLxF}Y_!-} zqYdG7ujVRJ`6nMu$;3)S^?ke$j!T!RnFc}KY!O?}d2fh-ua>!63A;wEaR66;GjSue zqb!wOk>Tq^tTEOK$)Vv&sM~U=`OdnY@rs!D1Wsp7c!KrSTm&DF^PWR1_!zsRs{!$< zumObE;K_w6hA5QAArsI;<*#uOZ=H+@qsk$D!cU_B`$%z|kF%JVfW)owYDPpav2(q5 z3+9^vXhTG>Wp!U$ZEbKB@??fKu7>&QVGR_ptXx(_AAdejyq)4xdV~?%-kSNE6+yT3 zNX|X#k{z-*k?j+sS|H@#rXdm0_kzQVBrrlteYZMfMU1_p1<4C&kJV&vsP-+^)rc(| z4$TzX_XiH@{PKwp<z1Wp}_5`c`Qe=x4$Ou(WgAFkn^`$`z2#KglhA`H3y@CJ%_3oc+(7clr)5$W+S>8 zzU)g~i2HL2Z{XM~$gbbpgkTHMf%QqmTt2LgzkT}@-w%%JgJ%rQ(myc9v3z@=+(m%y z(!j4m6~@1p#(ig%N20N3>yyTyhKyHPAa z8IN5fo10m=UMGHfjRvP%RoTb*YJ zOq_>lcRMgui-q~;rC#+7h~vmDjaHC6MkOR1?@&r1jEC4S1%#P(Y(t^X*I3gqu@zIP zI`IHeM`u~+3878hBlN;TXVx*Rz>|e?kNBK5zeXe8{l_2mqdF#yF6EZCGd(sxCZvW8 zSl-7%tK8uqRMxh)^+{*LSAji9R$EY>T}ZT7*s;e3kUr&tk}VP3BummXa~BeZ}<@CBKR% zqgwb?!XVw4%);HrZXnWdMLl+!x~1EtS|{FGJ!bT%W353;>gObqgUk$Yhl2*Fu&f?tj#VICh}8r=_XmizIXYg z-5#|4R`V`>Xu6^6kw|X8&+3j&%<*ofbhNBz2sB!2b8t3=~KzXR0B`tEw{_Vj?A-+zBoYTyq8R1_YGdJ`{ zH?j^`rjZtJu`xm`Flq3BwPJ;}CorHs;SObD7Ivd6ZW%#r+d3{0ex&mUEH5Nfx?efA zKKS0o0_vxC0&T@dHx?%ukf4_Ah>@Yi)^Gi?@I65p+Z@kz0Xg|q?k4@%@AaR$b&>?u znMUR@8vQ0IvaUKJ9M`>VA)9pR)G9wIEtFY99=|a; z?wy|85q%eaNf_kU-{Sw_dF%7!Kw&{QW0HQ`J-Tjb;TLfZjx~NN_5%;Vhq#M6ip8)f z-X(|`!eY~*YaigfrG$NW+SmjCxbiw#J7Rdi7qk;6JQWTrHhya}TmI$U_%qz#2gu_k zSt@bFaKW(&bcanXn#+%#%2P=r((5#aE2Ww@Rq_t6Eqh^skF!$jR_obnx4URXv_WY? z=lo(Zb1Eo9Aa$HteWujN1ngs6S>~$qn_TJLaPhSRe}V-kve(NU++9dE5IiT$Ru8WT z()Oeg$ROZq?(%T?G)J0F<+V5mF5%8;y0mK3deOY1XA|~`eoDc1@rGKMe_;e+YpT%X z)@w>3X43gge`bFtap!CMgNS5KJUSuqbi{A!!ehR+ZUS;{d3wIDGG<0r_|NfOEn`q zCsSN-(Ds9gw%A@${6(?*g90KQ`iT18rTshUw`LlX=xx87gx0(-Gcjx}Z!dXRCRkl! z3!o%kR;rFY?p#95hUxT(;|x6JHaf~jGGub$cuSkBzm1($gJ{KbdTRw2X*5Q9USA%I zY-3I|D1meqJWd<-<$3#l@iUt9Tz<*}7(p8JvJICIpR4ctcn4XTyWb^=NUp_@CoDjE zg7+Ax2cFqMP{ZX~A!@?~?4uk>toqKlP!U#KJ-=t?YgP(MOH1!+Z}v}4o7G{pV2*~I zF24$>4{x>5dFNzj+u90?1}EB~QomFb1Tc&w`o3cgIk414ntb3Whxt`zq&5*W^YHsD zH#h;f3V9}NaNh}PUj~u`t0YPcz#|y2heTe7WBF8k(?KMvXvX(r+Ht`y=Rb#NCaNBD z?;Zn=>e@eC?0hq$^jmwJyO}!CmppQcu4~fwE6KKfT%!2=dPfrSzU(v3yECjs`I|gG z>I&x00BtSGPtw&oN-^>2abaI=kB6?+ajW*d(irsU=OwmoUP2X3wQ`Ba_E}yJtz1uk zDuoND9R+^k)(p6t%Y^c5tUn9@6V=vwF?r9vZeAL5ggd@opBogIwNEA#)eqdhKEynU zKE{d<47}WexGdmBb2@XK&%|w|hO` zWS6l6D(W|>ynhSt$uD{S6XKn=_;|jI`xT!_mG&$BfCN{Yvr~n*G~mg(U!hffjiv4F zb6nfMmv50H77NDRxuH?;ejKVo?&ShJ!JELA9;I||gU{tOd>uS8Y67Lcf)j-%_BHgL zHx`hMUcofFxcv3*#Q4M4uaKu76?{|839CHdDy;$_Pi=je51}5`-!$gBdn2GW98b#? zVwztL&oC!=AaROEPLm==YBEt$qBs#0njqfVcJS`7D7Dq4ru4(_&FE8PF&HuEXQ1AO zSzX&nws54Ef11yF0YjG~aReL8kc9-fELT{_OxqpUMo5M=T+hXEF19g*do0Clghbo$ zl7qk{WCeR*)TU0zw5-QGY-Up$P--U>hoTgzsZOnS79C@RzR$7ttjKV}F=@CWY_U!f z!!0o#0ePx%ohE?QeyI^yL7zzmFOt9c$?`*eCZpz@W+vIDa2x=96FGw+@Ad9-Zz2^E zIoBy1l_&_+JYD%E!8XE|XA|>-W+M;-HR!0(cD`CAw|t>xnTB z@&FhTK^o#m1$4Zg_C7iuKoU+j#d~DQ)k!FW`Yz%1_L)m#pE1Ib%Y6Q?_RcaYs z!vK=fBHi7fbd7*?qaY2Ul%#adfRfVPEr=4*ohsca4T5wG-81jT>$>jezVG+>{C;}Z zV(m4DJVOOHP35IMY6Oyq76bJvYFI@r`bulkag-BFrnUmso0ZJ+dUi7$%& z^o19iywdfJR9@eYWN1~R#Ne}S{_J7}I&HX7F^bmirmR!TTG$ESt<{$~@4LL9S zbqtgI;rSY!^>Slo4?L=0HH-c*0SaZf&bIYd0wS}i&!R5cqm09?!RA~0mJ6W1p?}xt zp8UX2xcB6BHa0feH8B~FN%mpD&DB!tZ^x?PMo7bUg++(bLG|?6GHKanz=SSVCB1mR zgV6CWoQ>bRmo6yd#48Q!JbV$TItH~zwyYx|eTwTrB=U+Vf(|1DPt2F~Ig@+&)f(0` z@JOh%EIJsxyniiZ4F+rz+?;P@v-78YABS1c3$`BVq>HQTeR;_F3T<=j;|H_630$zg zzW!?=@G|;Rt7n36h_$`~{tx>s@AK@D&N6OdM_-%bK;MZ455?HRktd(2>N2F9wbv1E z8u>|1sKzwn#kbMO_3ZlI7B5#PbQ9UpCC%!9YMl-;6nGr$dP3PN5)>JG-MZ|}$yYt$8ulH~olUO= zV9N1@o4rJ;FkBJtabFtXFkcBD-b$MbG{_~Cj=SJ;^ zgsDeB)V+_|W1gPY?LT9@*}Gy*vmtQ$@ZMrvqqBhD{`pkYT3 ziZ;fk$}cu(%-Qb!2*0CjsGk~z^?OaX{^xRAUn&!{A7Mt&N<1TdT^7I4uy~;EIFL#Y zXfNV)`4O4ZFVnx|=j@~!7u-8??=6)T<7=L~zcbUVI(N1gE4(on2PnpThbp9zUo)mX zzIE9qaH@gX8=U90q}f3gi+h(#29Y2{LC&Tez;w~xxZn&=C!)cOFHS zFGwoiTXZG9mJI|koTY67-Klkh; zuZ6c3+Zv@6eo1NpLS@8fRv0*N%F{GW*m?3?T{RL+CI~M>!MF3=6hoH>v+c(iB2K4G zJsg1h`0@^Ah>q__Y;Kg+E!s{!F%NB3NOJ|UZ~6fd(iNacQhs(XIhim*R-IjKCUS=` zqOP^76>C<31l09FG3Jad1@j%G8>5K^nh!s5qsW2_19GDfqI9+|G8Nyuz;@q7+dJ45 zC!fcOL_THS)0>=&S-(pI5)KAgeKwF&oIA+>yiYu=0CoTaIm}rb!q}+Oqz#{j9D7zx zbeM>;#|490nrJ;kuKIu@$2R2Qv`L#}t|s#K+Oa=6`?gZ!G2yIrb{xlb^qQAoa(>n? zxGVlE5%_Ea6>Vdkz2g{K%hig{(V_Q2C}lIPCG9}$qHR5!+LWbz^03YKz_tk(XPsBu z@ocR?oc3;BY8~EkO{;P={wqxV8v74=lxMPPeBht0ITr`^xA*#BLknvPIwtXwAr4bn z-=waO@OpprC!^I+(%HxB4BW0D((kRJ7?(|N3y2?O#}IuY9-I34B9aAGjenQ9%?rKk zU+?yCBL{)D0=>JetQ?X%n|+7z9Z0VBM)IqaYAhJgPI7-SURl6*i^t;{d^5QaPFclR zPB+2Ap49ibtB4*Db89*CXSvF{`zg7skwk|@Pcl>tD$kY<%n=B}NEFp8PVZ;zbUKUD zGiwnUvU&yA0@7BB#J3MD%uNs%N15 z{(<9p4aASFNMO>an&4(!)x|{oho5A|+}DT`)s6`&CrVLUDzYNL#B#SUjWR{)1ceE9j|YhMj9^jMsInW>>Y zBbI%hxZ77kZ@Xnzd3gq&Mj!w(V4y=CTj|e zo$vHbZFm_rTqgeZtT#@rwOBG%peTfbE(2_-Rd!Bqr_e~XHtf+q8vm3fyrZQxYrCzn z(5hZfZ`+ClER}~2RUwj-!rfU(56A!EQ*P3-F8WwXOy0Z3L+;ViPcY+s&wGp*N70xc zz!r#)w^REb#R50skf#8(HfRlfz5_+ zalmY`2Q-J_lVZP)(`IOiX|wiv%Of2jE`57k9CstEVkH6FBkD1a?{qdlMV!h+nxIvm zNCh@)a|K*)tSvCV?f?rS)U8UXPZ`>6Py9BzznaiUxTmlhkNs9jehq~C{dJ@gAIOC4 zQwzp1%~H8@nd*ql!(Z?gXQCR^t$22vhLCEbxa}@B6fC@%uLTGJ{ae~&iqyOkEQZqy zCnSpY$tL^Gbb;-K`d6w4t8OY&2~FG^0g=^(?;cp~ft$*jj}@C)`})JXCw;+F@s~hC zdpxjvFbKw@=1^SFru^D*#K!{ zvQj*Y*L~_fngYE`_&0pnZT^GSkd;W9o{PZQ`=1yIBg}Bs13D=h&Tp}%^7-7JV;${? zqleo#1$BPoejyA|zaUEiZ^{%?PoB_m=F!JT>T(d{=?!@MM=ypRZGFk7t zsn&9nAT3lz6@FX0QQSi{y?$wmkZV0XDpxBFl2yR*TeQ;TqHv?1A5&$_(X5|guREvR zy|GSmZh>CZO|N}>{XuOF<`SY}Thbv~vA5H99pt;`!E2LNu#77*)2MbOJq#l3NX;>6vVMxCkRJn0yW%3|J}8c@^*ZKC8{ z)kkCxecWh6W+A36UGG+DB)G}~NMF5+z~>jghX?&4k)w6po0E-1M|E3DdP0ty(T!0R z@lZCSn|+#!RCD9X$w zTI}QHqvm`2)i;ljKIXNHbYjK%B73M5oE6CWe0ABJj(>~zhT$DSj5sR~+i^njWVAwn zkou^OMd!@$7mCkKFA>@Soit$77s9u5pF?pQ>{A9t1Akf*Ymt1gqJjc)z)_RTN zF#@Z_G*7-!Quh4@&TPCIaelLqG&!k4D6$@K>U!2bKMTAz@|2R*LJd#=WZ(?X&0=O+ z{hD&xKk7`C8FTW@)A`*$`pMWWnebWB5f%SU!Ne0Ogc}&xt z2yh$u+;Y7KH4GbjR5x*thVnBFUCH|_$cX0ysc!9wZgvRfuYWxnv*2EevtfbG?TIya zL-(T&&+?~zU9%BKy;p*P2I%SmMUvR$792`uu3pwB)XaO@#d9&+7{s`ApTYDW<@W+VYtv=gv_t29o_y zIJ(Nas3o0$$sK?L1j|AesT<$^{@l*k6xG@;j$Z6%0P&Ilr0u(tPj#Wqssef?=WN26 z*DuMpg77{ky9RK0$7);0aIjkl*5FG-8Cn%6H?z|;8|r>0`dNsttg4no=n?sVq&%oQ zSSodXFnI0J7_Q)G3|mnKuhhm3*;&8jvlY{Fe@XrGEnADwfWv$80fS#2F+YtI927LZ zKgLHd1D+-dN*Kv-j7$t*R)SCsY2~$|?59|o0kL3Eh ztpV%Y#H%065!4PNpQshfC>F4nyiLav_!-a%B2n-=j=fu)9OM|Nre6MpS3?Hzpz^SU zlV{_+WFBVqFs)AqUlCmd9?#o|od(k$+cO;6Vsd?v8z08%}#b&L`zyWBef)E*fl>fDl?G85^4VG90@Yo(}g#`Xml-3-4)xQ3imBQ zKiUv=Jbwe=c%t*DyTXcfiYQo#+14`dp_2)W><6Ob*byfRg~VMAi60~gFZqt|(7ek< zjhF5?M@e~B*bW+LgF<=paT5?iaoxuKpjDWbBXAo0JW07%)PFw2!IBL1>UT+lS#(F! zZ3@4|uELN~&KPF9+1YN@aPW`grZKvTm9MOPpWb+o&)0;MEUsi45M~`IX1kn!)iS}I zty52YxbLNjfOm+S49hnWODrw8q3+gJ4&*-TCQ*drdi{zDlaj}VUdt9QH9HfFKKGqk zvdnfP4#AB-`Ti>V)E&JGxiF1f)?u1)z<;a|yT5VZ1-Z}MI^-S@RXlJSYSuGN@+1X} zl?U3V-9o0`oMO->Hd_P*PDnX)RMFNju;t#R)Na1e^-;+bpDvqV0g;ws%f-JlZS$4P zgKp~5<6D5VPb2)y@?0r@8YcOlgstsKe+n9S{GQ@k6wQ=Clg`mNo*lz|FFqh`vFL;M zrhlYVG~xG#l2Ggeaw8&vvY7tAd-3HI0}Ltp_70^qlIu z2ULkQc<1N#=kx5xOWF3=Y6qG@(k#%xGM95|ib+0GGXj~+(2p*pK7zs!9k0Z0Oqe{w?<_dD3%@1{(&m3_drP(p z$|(t|6N~_9GRZu_;Wgpx;lwNVDM5}Uw=%T>y7XHFsQOyZ6_5E?8gYo9ZJFh>FB+eO zX(KOWKf6{XIi?HI@*kEwCmTKi=s3^VlD%`-NF}mk9$wc>o8WK^5{#L3bfC+>8)LxB z@>(ZP1r)X#au{XOES!8Gh&hJFTM~%-!|s~vptpF#SFD19tPw=;YF;JzXc}&hdFK#* zgNE_B1Xm-k+5qkd##-Hws{2JRsi*e4SGM~ghRn68C_4Upk`DsEgV8J5~5Fb zV7@{v0Bs?cmF}BuH(S-u>NY67KAhyA*{BRe<1u(iV%0_-OSC}`?Ah5k(*4(x44WTu zCxk%#o)Bq&m7@Q(#TO5ukamh$iz$NcoF<+rX*p3pCYcaz2xwUg)fH&#n(dR9ez#-B0Q@@r{FEsuWaTT zUcDB=#Z{u1snE)Q+CD{XGg_ceY%w38;G^3A0$TL>a3|&^#=Vb9(Tm)g?T)$@)0e-F zO>Vnn2?+53(#t#PgfZRw*lYz6E(Z%0b0HOy5Az<3V8~&^eCTXG+DAe!c=QE)VlyQ6 zZ;67OCwQ6Fp5?gjT`AVH?!FLZ4M`o# z3a|r1q;bVlgx#J(#G;=b&Xfb$63+}8jAc~t&NI}}(NPfuFK6)W(k)5n zkuFc}lT6HzNqBy>nLfkczY`ZVR!CzdVU+h~1r`$wuSpTQPLcF{);Y-M?0n(NH}$NC=TwS(cqbkcpR`!0^>c3P zv;KlaouI|!t>5JrwGCzMPl}9f-0ZhNdxx`;X{_kg3%NdkR_|6AkHfOcW?CHPVpWj5HC_Bcg3q?Z zO!P%ZlV(ff>6$R|jN8lckoEId69;Wn(P#w&`fBk?52qnXw!G_7g{+vBma87zQazBP zr$=8p%l*^ynp*KZ57SgW&lj3nI73oFZyQ_;8YQ6TsFmuXjtfn5RySuxo=`;P*8*APEmeR|K77< z4{>M^pOYzi6OFu$)AN2%vRkm1)3W|KC%FBC*O0VyLPn;~pw))-{gWqdP3w*agBa@Rg}&Z`*C3VWIFvy64Rs4X;z&UG?G)92>VZ8w0w6=eC+pem6{G zfqmQE-W}pjlK5sR!77- zujMfmv9G|q*JWQ5)6Q9C*(2`Q#yx7Kd?r*Dp8JH}XWH0<9Q7LD7ruVmGI#vT=NA-t z4psAaEO;1T<;i!n{iR%4e}}pSakV4W&>73JM!9qmEXo`~*z?0=;ixM6wxqFX`TWMS zTPB=!Z9+lIm@R5WEgGxOb!`rR?ed`R$+WUu6dkP;gMb+wETD^H%eX0Ni!mIPaT_g) zSrgBqv#OdU)GPUUkT@VHe!=%Xz`_kDkA%QE;0+Nmu?36g-RoO7 zUF!Qk(6+LQAHDtUxOAgIJ~C>EMY?Z+e|}gxm&mPwT>q#{X(=oCK5B1V194Eip-OWm zx-;E(zIZaUVq(Ku9vsYyOU8se{<)#XbGY>-sh?Nc6VOI`QijM*mMSXtfo@awk zUB!Pq9O~!y^Uns7Xv2rIE-t5iO!qhxNK4xqmo;b49;Yb)gwy_r1ITC$M#E%U)Z z2ii0dAw`M|yQVy&u&+q3V2j5*I7qzkGpUg<85LiU47`rwQ;rMUtODhVr^fs6`*ZXG zeGFfLus1+{BsyI5IS>po*lU2ljb_MVBM}Dxit$JZPR@P@ycw#jb%4}$YKy;Q{z(Zm z17y(UC~`9Mp|D+OeSN6&8Z)6D9@tHyToI1ctR??kJJ4ijl<|G*Sv*=WHAz5OFE}5I zjZbT!ZOQg<6Q2Q1R+2vh}) z&0ahieXW{HYV_@WNK&@nvkB5jk$s6p0Uw{>64Ocp?BgT91h7e@E*!Oa zJvfrKR6aZ+ox9i^)g~mS@LsO*Kd&0Vv;y;O-fe4yhYi<`v-r#dhn-CQFJC|Ljy7rW zPVLN>5`Sifj*H8wTLQO5@mkTkAq|{G+nFvupB=*g7oqg;4+K$YDR^vD<+j`u<1;!| z&x)Zx#w`la5B2I#e_i5qy+drSfb$h;UFZ08ctPee@~(?A9}e6{Pnh(|O`5VSH@k#L za*d^Ya!Ee@q6K7s+U>%2zLK;pMY4c`jeb>^TWa1#QF>4b`smzlC9;)KSo9yd9Ya7n zuvSpz9|mBLeZ(3P^l*d%jVtGiEPjh%c7~fC#^@e^B_yN$>A~f1==I#v71_=8$(9B7 zbSLDnTN*J+MrUU#@)z>IWE$Tb<5>`FKJv{02)IALcycA|>mEi? zp!9EWC6yXnoIg_77qUusZ^-h#m$!TX0Bo2HKEx&IQP}d&tq^x9MtAtBRXm#;ViuI5 z3#Zq(PsB`4zl-<>)d&$GXve?ySn|_r^f=&wchXAxH{WT`9zB-$gF4L=bBhCr5B<>F zX=rL-3me4Hq(c;WS`O_!T&3{Zhyl23{m`3kXv&G~rw3m>4_|ic^8AYtT#r?)p&pA7 z5}gpmUc9eHe|a;h=g<4pij()LqoP=^{2h~yPW6t#&GN9a7H>69mtYv+=^ZMSmT?|T z`be`#smF~Q^gkKQq*XM-eIa_FqciXEi4;@aKAR`<84CXE&W0*mwvoKH9n#Vm9fMaQ zIb+7dK@?zMzyJJF+CV_8=9mP(%WG7Z&jc$}-nt}cLKEbfBhOatFxU0P_V$6Aa7;F6C4~0MZUg(y$$$PF!Gn*nbL?MM zo(x#;w9unv!+n}68E|DrB!&JUY0VV?5#`sWf*MDNPu**kTo zjF|p5rYGJx%Jpl*dW z%b6;U2$5Vk6z2zhkhBSPwUB+zMt7@moWf@D%xOP*)x&Xy{R=6Vpcd|sz0p4?c11cE z?IqiY_{;B@I#d^z8`;7%$Yd%(e{39_$Y01G4^0z*W9Z&!=#9nY4_<%zBK2baHrzm0 znS6u zm|?ra!(0IG4uP3+8R;jY-4PO3FBO=TT9EXLgX3j}3h_s*RvU}jEgQK;KWjw$RX?6K zl~GWB}(R1D~b%JB~ETlk(Q9&w$L&dv5~)b4S2Kq%X)oW~#TH z4_oPgVC;G-9xBAdB_Sk2q!<&>qDooZ=qIeiE5lv;~>p4 z&aQWN4FGtTLoBHI;BFGb2}dvp9NeiMzjt5Dk3Gpn4m$dev+WQQ6RXL!KN^-`#KOXo zdQoP*-3FYH_W%J|6?P7eqZ$8hYLA}xSj4ImOWTuQUIXWAt@r%1^Xmiu?goF|=I%}( zK#VAzJKj^Xqjbs0F?}3nDQ2z2-3m)8MI5x>e&g$duU4uXv@eizP18TwzC4DlMw85H zXK+;JKyaS&kQVSf)6z={dMf0qDwB&O$1&P+Kl~ZTR_HbyjbP zZb&3P^r2qr;Xm)Q!13qpCDz$hhLst9-h{p1pMlbOWKZZmSE-u8?uo#2y_U)!IF00+ z!%7`}pWdBfL8%JP%_}h)N-6#DxsOw>4pC8q>%Stg(2Al#2LpjI>6R@5fMsCs-KM|1 z`6zr7)V6<%98=*{($U$_4{j8UVF9~R(i%(VTwhwfdgZFzx#MxsbkNEPu!j*d0k?Os zcEJ7?f|L{#YXHXKxPmK}{!hXGTo!O*&^8x?t?s>8R#wa06?k`>+K=Qo#FE`>>P{g) zS0e82Uov??k8w_trhgS={@|w?CT@JHrFia=)M4nR-@DKnh)?t>j+jXgH5vf~gn~x(VB}YeVVy(b z>;9w}dvD!x6ML(p9f+Ux8wZD^X%inc%fU2eU~GIqJuzLlG}F+~Pzem14s~avji?4X zbW^?_!a_o`1@ForIsf@oJlHX4P8HMnk1aDt_53SU-%4F-uS=b>l79{ZJz6!XJ02}) z-wkqk9ZWtU?Y|kCkS2NUw(j4Aw}|}t*c_fdn!mdpfMKZ`-@f31p1plhYR3S(F86;Ies$KsdS6zwx`g@1d-WVSf}XW@ zGYpLK5ivnH50$;W8=EhFeQ*zgOf$5LEoWzE^9BV4Rho5xQ-LKDfR0z|>+5U$jl%*0 z2f#KjU2EEbW9vrAEY?5P`q!lm@EP0(--oO9tkzxh%&nqjTOm7x+atX+K+N(mJ>bH; zu>Rni3vHe7!rP+a)+erfY7zp60*rZuK@V>Uo}QEBBsMyHi-72GohZ~Wf!x<*mD!l zepN-SIX`ldZRBNVRKgDTsq$~v$inr?srmSnqkl-HH{e_C#n**$>9xzOeHOen6`IuZ zdVO@h)2UpRVomqmgbWT*uO0sVzg-Xc2`()yJp#`2L-F9l>B1Ua(X@_l&;UK+lPFqg znlrBxh_PTwYAQRRK~A7AEQ~1wjiAA%aQ>_5lmW(dx+Ny~c(?;=w|UfV;!!;1aPWxU zEpA@eK~B0x}W5AFOn47Ig$JWcjiRtONC%y~kPyn)dG-c?hq@@*WotIOTl#;?WEOlLC ztd-MkYadXzl=mqT{kZtYJS7(+TTbCyARlU zQwhcd(U6gV8lsm?nIxf#UJsO9Yc~wz_Uza{8?$bb)acB_5@2H~T)nDREXTXRrxB(4?=V>AXdno0pvvL_-j?=xZ9`f7xMR)jN?0Z%L^KhUc%}B zZWXZha)JvWuD`d71YQrIx#(27UzF0W+Z(A<;bP{BM7iHN9X=9{dce%>NiiK*n-t=|i``py_T+xR&B zBT~3*DJBeGJvPKY*`XFYylQmo_*^5ME5B=zRcCyB&1j3fm`&VDmwY00fubkF)CwDy zI*E?4d!Dm5UYltP;e6IAMLL_dP5IIKT#EhnEM=jU?nHF|FIT`IOK7udnVA6IT|Bhy z!5K#IF+o`|+RHeTU-ZCCk9>MF$;OVjT( zJfp0Uj@#beA1hV_=wAU+cLVq%YF;X96t#PsT!^UDn{}K;9*f7)YT>31De1MdpKZk< zWk2Np|TaYCRJix~5`2yCdtrQXt7-9uUol)%2GUH-@|EnO%fu}&fmQ7+(nQ~ucK z3;xbWs`q!z11P1f7#ue6>e;izk9VF zvzD5A)aVwq@UqUMD0DX8ZPfoda;NqlEW1`T+`x$k7X?j%8A8;qv+(1`+4JRQ@{#G6 z?FNi=PI`IlJQ-%X_fqmYWgUim>&^YOtltPeLJku3%In8S+{d<-l;loQ=`iw;MwN>Kf%mb1(Y|~*Gx7xnep-5KopABnq=5Fn{Qbx2 z|8{AoM?u;l)-{U=IsSgkzdG=rUi{Bx1v3T>Qz6Xaf2O|w{shkfDS`0aDUu`pcJJTg zz&|AbUm+sENY|tyHrjh85H0wxe^L?EpI8vldA$sm)gu1ei;E9M47?Wa@xSRqUBnN@ z6vG-R$`rZ4xO*0eG~mrmjAzmrZ{53SZwc=4|K;)J1Q;HJFYUvfX9H*-07KQ&!AB8( zLaO&Z+|JFNltQnQdI%ux_5s{8;@s_fCN*kA?R&%Yqkmi_!sJxH{N~bfXT++T?!fUsiw%M}K zYZ3<<^2ddW!q=48EA1xxOr=7t{xn+&MnJ74Mg$^1mT*}xvOYi;Lip7fVd3uF;&`F| zd8i4jO;X4k=|#)(orZy;wacftBnrl?Pd5Ye%wcRYxlR;+Y{lqQu$zFUhaxAN=fT$! z2=M@?VW!yM3vaz*C~A`Rq`GTVbm8cMPhoE8?LUr%alkvsof2^9FZB;2A4`$l6kZ5Z zWWs2T9;2(g`n$W<#1{h=yFiX2z*ti#Bj+qAd@+(vK8q?CVd3g!O_Gj7O@Ee~a&mMc z7p8V`pxMV^&?Y^UO}F{OcQ)Q3!9Wd6{aVxIARZzTK+nbW1U81S5HU`0vVrT-d%KHW z;^H3OYO4>9HXqvcMV~^kGRVa%+tdfW2qEpOU@@nhFU{8bm?cYxVV8Kwa&Z)5_B)&E zpxh#&g3(|{acJEvYAFxroT@y$?w4t{Z!5LNIV$kx*_1GVXU6G_Ouhn2&Gww)R(l0XH`fSoZL&E?s8C zT84+jjr{L}3vjO+Ziu-V^qhC}Kld{K)-^@AfKIL6zeFHix)xNLHf3DB}RRA!})q#{i|Gyf4oB02gHINv@?CMn+(uYyNORPakPu1nh IWXuBp54T1>GXMYp literal 0 HcmV?d00001 diff --git a/mkdocs.yml b/mkdocs.yml index 1c7801191..dc9b5a469 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -29,6 +29,9 @@ markdown_extensions: # - pymdownx.mark # - pymdownx.smartsymbols - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.details # - pymdownx.tasklist: # custom_checkbox: true #- pymdownx.tilde @@ -162,6 +165,7 @@ theme: - toc.follow # - toc.integrate - content.code.copy + - content.tabs nav: - 🏠 Home: index.md @@ -227,6 +231,8 @@ nav: - Where to Log: trulens_eval/tracking/logging/where_to_log/index.md - ❄️ Logging in Snowflake: trulens_eval/tracking/logging/where_to_log/log_in_snowflake.md - πŸ““ Logging Methods: trulens_eval/tracking/logging/logging.ipynb + + - πŸ›‘οΈ Guardrails: trulens_eval/guardrails/index.md - πŸ” Guides: # PLACEHOLDER: - trulens_eval/guides/index.md - Any LLM App: trulens_eval/guides/use_cases_any.md @@ -260,6 +266,10 @@ nav: - trulens_eval/api/endpoint/index.md - OpenAI: trulens_eval/api/endpoint/openai.md - 𝄒 Instruments: trulens_eval/api/instruments.md + - πŸ›‘οΈ Guardrails: + - trulens_eval/api_guardrails/index.md + - πŸ¦œοΈπŸ”— Langchain Guardrails: trulens_eval/api/guardrails/langchain.md + - πŸ¦™ Llama-Index Guardrails: trulens_eval/api/guardrails/llama.md - πŸ—„ Database: - trulens_eval/api/database/index.md - ✨ Migration: trulens_eval/api/database/migration.md diff --git a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb index 3a897e4cd..4cf05281c 100644 --- a/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb +++ b/trulens_eval/examples/expositional/end2end_apps/trubot/trubot_example.ipynb @@ -118,7 +118,7 @@ "from trulens_eval import Select\n", "from trulens_eval import TP\n", "from trulens_eval import Tru\n", - "from trulens_eval.utils.langchain import WithFeedbackFilterDocuments\n", + "from trulens_eval.guardrails.langchain import WithFeedbackFilterDocuments\n", "\n", "pp = PrettyPrinter()\n", "\n", diff --git a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb index b717c5c23..84e7ef125 100644 --- a/trulens_eval/examples/quickstart/langchain_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/langchain_quickstart.ipynb @@ -7,7 +7,11 @@ "source": [ "# πŸ““ _LangChain_ Quickstart\n", "\n", - "In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response.\n", + "In this quickstart you will create a simple LCEL Chain and learn how to log it and get feedback on an LLM response.\n", + "\n", + "For evaluation, we will leverage the RAG triad of groundedness, context relevance and answer relevance.\n", + "\n", + "You'll also learn how to use feedbacks for guardrails, via filtering retrieved context.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/langchain_quickstart.ipynb)" ] @@ -24,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -33,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -51,13 +55,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/snowday/lib/python3.11/site-packages/_distutils_hack/__init__.py:26: UserWarning: Setuptools is replacing distutils.\n", + " warnings.warn(\"Setuptools is replacing distutils.\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "πŸ¦‘ Tru initialized with db url sqlite:///default.sqlite .\n", + "πŸ›‘ Secret keys may be written to the database. See the `database_redact_keys` option of `Tru` to prevent this.\n" + ] + } + ], "source": [ "# Imports main tools:\n", "from trulens_eval import TruChain, Tru\n", "tru = Tru()\n", + "tru.reset_database()\n", "\n", "# Imports from LangChain to build app\n", "import bs4\n", @@ -78,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -128,9 +150,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/snowday/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n", + " warn_deprecated(\n" + ] + } + ], "source": [ "retriever = vectorstore.as_retriever()\n", "\n", @@ -158,9 +189,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Task Decomposition is a technique that breaks down complex tasks into smaller and simpler steps to enhance model performance. It involves transforming big tasks into manageable tasks and exploring multiple reasoning possibilities at each step. Task decomposition can be done using simple prompting, task-specific instructions, or by relying on an external classical planner for long-horizon planning.'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "rag_chain.invoke(\"What is Task Decomposition?\")" ] @@ -175,9 +217,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In Groundedness, input source will be set to __record__.app.first.steps__.context.first.invoke.rets[:].page_content.collect() .\n", + "βœ… In Groundedness, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In Answer Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In Answer Relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In Context Relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In Context Relevance, input context will be set to __record__.app.first.steps__.context.first.invoke.rets[:].page_content .\n" + ] + } + ], "source": [ "from trulens_eval.feedback.provider import OpenAI\n", "from trulens_eval import Feedback\n", @@ -192,19 +247,19 @@ "\n", "# Define a groundedness feedback function\n", "f_groundedness = (\n", - " Feedback(provider.groundedness_measure_with_cot_reasons)\n", + " Feedback(provider.groundedness_measure_with_cot_reasons, name = \"Groundedness\")\n", " .on(context.collect()) # collect context chunks into a list\n", " .on_output()\n", ")\n", "\n", "# Question/answer relevance between overall question and answer.\n", "f_answer_relevance = (\n", - " Feedback(provider.relevance)\n", + " Feedback(provider.relevance_with_cot_reasons, name = \"Answer Relevance\")\n", " .on_input_output()\n", ")\n", - "# Question/statement relevance between question and each context chunk.\n", + "# Context relevance between question and each context chunk.\n", "f_context_relevance = (\n", - " Feedback(provider.context_relevance_with_cot_reasons)\n", + " Feedback(provider.context_relevance_with_cot_reasons, name = \"Context Relevance\")\n", " .on_input()\n", " .on(context)\n", " .aggregate(np.mean)\n", @@ -221,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -232,96 +287,411 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Task Decomposition is a technique that breaks down complex tasks into smaller and simpler steps to enhance model performance. It involves transforming big tasks into manageable tasks and exploring multiple reasoning possibilities at each step. Task decomposition can be done using simple prompting, task-specific instructions, or by relying on an external classical planner for long-horizon planning.'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "response, tru_record = tru_recorder.with_record(rag_chain.invoke, \"What is Task Decomposition?\")" + "with tru_recorder as recording:\n", + " llm_response = rag_chain.invoke(\"What is Task Decomposition?\")\n", + "\n", + "display(llm_response)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "json_like = tru_record.layout_calls_as_app()" + "Check results" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
Chain1_ChatApplication1.00.90.551.00.004991
\n", + "
" + ], + "text/plain": [ + " Groundedness Answer Relevance Context Relevance \\\n", + "app_id \n", + "Chain1_ChatApplication 1.0 0.9 0.55 \n", + "\n", + " latency total_cost \n", + "app_id \n", + "Chain1_ChatApplication 1.0 0.004991 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "json_like" + "tru.get_leaderboard()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By looking closer at context relevance, we see that our retriever is returning irrelevant context." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0What is Task Decomposition?Fig. 1. Overview of a LLM-powered autonomous a...0.8
1What is Task Decomposition?Fig. 10. A picture of a sea otter using rock t...0.4
2What is Task Decomposition?(3) Task execution: Expert models execute on t...0.6
3What is Task Decomposition?Fig. 6. Illustration of how Algorithm Distilla...0.4
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 What is Task Decomposition? \n", + "1 What is Task Decomposition? \n", + "2 What is Task Decomposition? \n", + "3 What is Task Decomposition? \n", + "\n", + " context ret \n", + "0 Fig. 1. Overview of a LLM-powered autonomous a... 0.8 \n", + "1 Fig. 10. A picture of a sea otter using rock t... 0.4 \n", + "2 (3) Task execution: Expert models execute on t... 0.6 \n", + "3 Fig. 6. Illustration of how Algorithm Distilla... 0.4 " + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from ipytree import Tree, Node\n", + "last_record = recording.records[-1]\n", "\n", - "def display_call_stack(data):\n", - " tree = Tree()\n", - " tree.add_node(Node('Record ID: {}'.format(data['record_id'])))\n", - " tree.add_node(Node('App ID: {}'.format(data['app_id'])))\n", - " tree.add_node(Node('Cost: {}'.format(data['cost'])))\n", - " tree.add_node(Node('Performance: {}'.format(data['perf'])))\n", - " tree.add_node(Node('Timestamp: {}'.format(data['ts'])))\n", - " tree.add_node(Node('Tags: {}'.format(data['tags'])))\n", - " tree.add_node(Node('Main Input: {}'.format(data['main_input'])))\n", - " tree.add_node(Node('Main Output: {}'.format(data['main_output'])))\n", - " tree.add_node(Node('Main Error: {}'.format(data['main_error'])))\n", - " \n", - " calls_node = Node('Calls')\n", - " tree.add_node(calls_node)\n", - " \n", - " for call in data['calls']:\n", - " call_node = Node('Call')\n", - " calls_node.add_node(call_node)\n", - " \n", - " for step in call['stack']:\n", - " step_node = Node('Step: {}'.format(step['path']))\n", - " call_node.add_node(step_node)\n", - " if 'expanded' in step:\n", - " expanded_node = Node('Expanded')\n", - " step_node.add_node(expanded_node)\n", - " for expanded_step in step['expanded']:\n", - " expanded_step_node = Node('Step: {}'.format(expanded_step['path']))\n", - " expanded_node.add_node(expanded_step_node)\n", - " \n", - " return tree\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'Context Relevance')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use guardrails\n", "\n", - "# Usage\n", - "tree = display_call_stack(json_like)\n", - "tree" + "In addition to making informed iteration, we can also directly use feedback results as guardrails at inference time. In particular, here we show how to use the context relevance score as a guardrail to filter out irrelevant context before it gets passed to the LLM. This both reduces hallucination and improves efficiency.\n", + "\n", + "Below, you can see the TruLens feedback display of each context relevance chunk retrieved by our RAG." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Wouldn't it be great if we could automatically filter out context chunks with relevance scores below 0.5?\n", + "\n", + "We can do so with the TruLens guardrail, *WithFeedbackFilterDocuments*. All we have to do is use the method `of_retriever` to create a new filtered retriever, passing in the original retriever along with the feedback function and threshold we want to use." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In context_relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In context_relevance, input context will be set to __record__.app.first.steps__.context.first.invoke.rets[:].page_content .\n" + ] + } + ], "source": [ - "tree" + "from trulens_eval.guardrails.langchain import WithFeedbackFilterDocuments\n", + "\n", + "# note: feedback function used for guardrail must only return a score, not also reasons\n", + "f_context_relevance_score = (\n", + " Feedback(provider.context_relevance)\n", + " .on_input()\n", + " .on(context)\n", + ")\n", + "\n", + "filtered_retriever = WithFeedbackFilterDocuments.of_retriever(\n", + " retriever=retriever,\n", + " feedback=f_context_relevance_score,\n", + " threshold=0.5\n", + " )\n", + "\n", + "rag_chain = (\n", + " {\"context\": filtered_retriever | format_docs, \"question\": RunnablePassthrough()}\n", + " | prompt\n", + " | llm\n", + " | StrOutputParser()\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we can operate as normal" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ + "tru_recorder = TruChain(rag_chain,\n", + " app_id='Chain1_ChatApplication_Filtered',\n", + " feedbacks=[f_answer_relevance, f_context_relevance, f_groundedness])\n", + "\n", "with tru_recorder as recording:\n", " llm_response = rag_chain.invoke(\"What is Task Decomposition?\")\n", "\n", "display(llm_response)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See the power of context filters!\n", + "\n", + "If we inspect the context relevance of our retreival now, you see only relevant context chunks!" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0What is Task Decomposition?Fig. 1. Overview of a LLM-powered autonomous a...0.8
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 What is Task Decomposition? \n", + "\n", + " context ret \n", + "0 Fig. 1. Overview of a LLM-powered autonomous a... 0.8 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "last_record = recording.records[-1]\n", + "\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'Context Relevance')" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting dashboard ...\n", + "Config file already exists. Skipping writing process.\n", + "Credentials file already exists. Skipping writing process.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c03498a4bf944abf8e93cd957cebe2ad", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dashboard started at http://192.168.4.206:1236 .\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tru.run_dashboard(port=1236)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -331,9 +701,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Record(record_id='record_hash_b5b6dc830a55b37fdffb2bbfa95df066', app_id='Chain1_ChatApplication_Filtered', cost=Cost(n_requests=6, n_successful_requests=6, n_classes=0, n_tokens=5171, n_stream_chunks=0, n_prompt_tokens=5102, n_completion_tokens=69, cost=0.007782000000000001), perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 276380), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 586834)), ts=datetime.datetime(2024, 6, 13, 10, 50, 31, 586866), tags='-', meta=None, main_input='What is Task Decomposition?', main_output='Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', main_error=None, calls=[RecordAppCall(call_id='6514d02b-7304-42bb-ab03-4b9cca2f35cf', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.question, method=Method(obj=Obj(cls=langchain_core.runnables.passthrough.RunnablePassthrough, id=13885082512, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13917059088, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='What is Task Decomposition?', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 399239), end_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 420463)), pid=65698, tid=6453375), RecordAppCall(call_id='87452040-fbc5-4522-a32d-1fcf7d317761', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=langchain_core.vectorstores.VectorStoreRetriever, id=13914853328, init_bindings=None), name='_get_relevant_documents'))], args={'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\", 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 442963), end_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 818347)), pid=65698, tid=6453374), RecordAppCall(call_id='c7f75d9a-e84c-4215-b4c3-a6c3497cc52a', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\"]}, rets=0.4, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 828037), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 276151)), pid=65698, tid=6453383), RecordAppCall(call_id='4c6ed35c-e01b-4730-868b-9f72e5cdba02', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.']}, rets=0.4, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 828982), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 281506)), pid=65698, tid=6453384), RecordAppCall(call_id='847a1d8e-4451-4fda-ba95-b799dbcc3dd0', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.']}, rets=0.7, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 822204), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 281915)), pid=65698, tid=6453381), RecordAppCall(call_id='e14d5a31-4785-457f-8c83-42d02d9552f1', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.']}, rets=0.2, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 825462), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 83574)), pid=65698, tid=6453382), RecordAppCall(call_id='e9a3cf02-7908-43b4-a5d7-cde30d4d82fd', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents'))], args={'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 425547), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 84178)), pid=65698, tid=6453374), RecordAppCall(call_id='229b7215-1669-45dd-91c2-0cc92410d191', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6415017872, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 402275), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 84741)), pid=65698, tid=6453374), RecordAppCall(call_id='cddb2f39-406a-43b3-abfb-beb4dec85349', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914646800, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 383316), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 86065)), pid=65698, tid=6453374), RecordAppCall(call_id='cbc5fbc3-e7d9-4ef4-9dd0-1f7ff6affe8f', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13871448208, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 336283), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 86599)), pid=65698, tid=6452996), RecordAppCall(call_id='1cde13e5-7d19-4a35-9e65-410d94ff11b4', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.middle[0], method=Method(obj=Obj(cls=langchain_core.prompts.chat.ChatPromptTemplate, id=13881883472, init_bindings=None), name='invoke'))], args={'input': {'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6418097808, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 122172), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 152428)), pid=65698, tid=6452996), RecordAppCall(call_id='469a1580-267e-4bcc-b8a1-f38b8fe0c04f', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.middle[1], method=Method(obj=Obj(cls=langchain_community.chat_models.openai.ChatOpenAI, id=13736902928, init_bindings=None), name='invoke'))], args={'input': {'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946206736, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 184183), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 530865)), pid=65698, tid=6452996), RecordAppCall(call_id='4e9dee49-5efc-4ba4-8d37-c628a4d68cea', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.last, method=Method(obj=Obj(cls=langchain_core.output_parsers.string.StrOutputParser, id=6414890448, init_bindings=None), name='invoke'))], args={'input': {'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946133968, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 562307), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 586799)), pid=65698, tid=6452996), RecordAppCall(call_id='3ad4449b-edbf-4ff3-9d61-87c6551fde9e', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?'}, rets='Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 276380), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 586834)), pid=65698, tid=6452996)], feedback_and_future_results=[(FeedbackDefinition(Answer Relevance,\n", + "\tselectors={'prompt': Lens().__record__.main_input, 'response': Lens().__record__.main_output},\n", + "\tif_exists=None\n", + "), ), (FeedbackDefinition(Context Relevance,\n", + "\tselectors={'question': Lens().__record__.main_input, 'context': Lens().__record__.app.first.steps__.context.first.invoke.rets[:].page_content},\n", + "\tif_exists=None\n", + "), ), (FeedbackDefinition(Groundedness,\n", + "\tselectors={'source': Lens().__record__.app.first.steps__.context.first.invoke.rets[:].page_content.collect(), 'statement': Lens().__record__.main_output},\n", + "\tif_exists=None\n", + "), )], feedback_results=[, , ])" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# The record of the app invocation can be retrieved from the `recording`:\n", "\n", @@ -345,11 +734,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Answer Relevance 0.8\n", + "Context Relevance 0.8\n", + "Groundedness 0.9666666666666667\n" + ] + } + ], "source": [ - "# The results of the feedback functions can be rertireved from\n", + "# The results of the feedback functions can be rertrieved from\n", "# `Record.feedback_results` or using the `wait_for_feedback_result` method. The\n", "# results if retrieved directly are `Future` instances (see\n", "# `concurrent.futures`). You can use `as_completed` to wait until they have\n", @@ -364,22 +763,253 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
app_idapp_jsontyperecord_idinputoutputtagsrecord_jsoncost_jsonperf_jsontsGroundednessAnswer RelevanceContext RelevanceGroundedness_callsAnswer Relevance_callsContext Relevance_callslatencytotal_tokenstotal_cost
0Chain1_ChatApplication{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...RunnableSequence(langchain_core.runnables.base)record_hash_ec83de0a61511aa3a885f0ce22ed1b89\"What is Task Decomposition?\"\"Task Decomposition is a technique that breaks...-{\"record_id\": \"record_hash_ec83de0a61511aa3a88...{\"n_requests\": 2, \"n_successful_requests\": 2, ...{\"start_time\": \"2024-06-13T10:50:05.859470\", \"...2024-06-13T10:50:07.8200271.0000000.90.55[{'args': {'source': ['Fig. 1. Overview of a L...[{'args': {'prompt': 'What is Task Decompositi...[{'args': {'question': 'What is Task Decomposi...133110.004991
1Chain1_ChatApplication_Filtered{\"tru_class_info\": {\"name\": \"TruChain\", \"modul...RunnableSequence(langchain_core.runnables.base)record_hash_b5b6dc830a55b37fdffb2bbfa95df066\"What is Task Decomposition?\"\"Task decomposition is a technique used to bre...-{\"record_id\": \"record_hash_b5b6dc830a55b37fdff...{\"n_requests\": 6, \"n_successful_requests\": 6, ...{\"start_time\": \"2024-06-13T10:50:27.276380\", \"...2024-06-13T10:50:31.5868660.9666670.80.80[{'args': {'source': ['Fig. 1. Overview of a L...[{'args': {'prompt': 'What is Task Decompositi...[{'args': {'question': 'What is Task Decomposi...151710.007782
\n", + "
" + ], + "text/plain": [ + " app_id \\\n", + "0 Chain1_ChatApplication \n", + "1 Chain1_ChatApplication_Filtered \n", + "\n", + " app_json \\\n", + "0 {\"tru_class_info\": {\"name\": \"TruChain\", \"modul... \n", + "1 {\"tru_class_info\": {\"name\": \"TruChain\", \"modul... \n", + "\n", + " type \\\n", + "0 RunnableSequence(langchain_core.runnables.base) \n", + "1 RunnableSequence(langchain_core.runnables.base) \n", + "\n", + " record_id \\\n", + "0 record_hash_ec83de0a61511aa3a885f0ce22ed1b89 \n", + "1 record_hash_b5b6dc830a55b37fdffb2bbfa95df066 \n", + "\n", + " input \\\n", + "0 \"What is Task Decomposition?\" \n", + "1 \"What is Task Decomposition?\" \n", + "\n", + " output tags \\\n", + "0 \"Task Decomposition is a technique that breaks... - \n", + "1 \"Task decomposition is a technique used to bre... - \n", + "\n", + " record_json \\\n", + "0 {\"record_id\": \"record_hash_ec83de0a61511aa3a88... \n", + "1 {\"record_id\": \"record_hash_b5b6dc830a55b37fdff... \n", + "\n", + " cost_json \\\n", + "0 {\"n_requests\": 2, \"n_successful_requests\": 2, ... \n", + "1 {\"n_requests\": 6, \"n_successful_requests\": 6, ... \n", + "\n", + " perf_json \\\n", + "0 {\"start_time\": \"2024-06-13T10:50:05.859470\", \"... \n", + "1 {\"start_time\": \"2024-06-13T10:50:27.276380\", \"... \n", + "\n", + " ts Groundedness Answer Relevance \\\n", + "0 2024-06-13T10:50:07.820027 1.000000 0.9 \n", + "1 2024-06-13T10:50:31.586866 0.966667 0.8 \n", + "\n", + " Context Relevance Groundedness_calls \\\n", + "0 0.55 [{'args': {'source': ['Fig. 1. Overview of a L... \n", + "1 0.80 [{'args': {'source': ['Fig. 1. Overview of a L... \n", + "\n", + " Answer Relevance_calls \\\n", + "0 [{'args': {'prompt': 'What is Task Decompositi... \n", + "1 [{'args': {'prompt': 'What is Task Decompositi... \n", + "\n", + " Context Relevance_calls latency total_tokens \\\n", + "0 [{'args': {'question': 'What is Task Decomposi... 1 3311 \n", + "1 [{'args': {'question': 'What is Task Decomposi... 1 5171 \n", + "\n", + " total_cost \n", + "0 0.004991 \n", + "1 0.007782 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "records, feedback = tru.get_records_and_feedback(app_ids=[\"Chain1_ChatApplication\"])\n", + "records, feedback = tru.get_records_and_feedback(app_ids=[])\n", "\n", "records.head()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
Chain1_ChatApplication1.0000000.90.551.00.004991
Chain1_ChatApplication_Filtered0.9666670.80.801.00.007782
\n", + "
" + ], + "text/plain": [ + " Groundedness Answer Relevance \\\n", + "app_id \n", + "Chain1_ChatApplication 1.000000 0.9 \n", + "Chain1_ChatApplication_Filtered 0.966667 0.8 \n", + "\n", + " Context Relevance latency total_cost \n", + "app_id \n", + "Chain1_ChatApplication 0.55 1.0 0.004991 \n", + "Chain1_ChatApplication_Filtered 0.80 1.0 0.007782 " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tru.get_leaderboard(app_ids=[\"Chain1_ChatApplication\"])" + "tru.get_leaderboard(app_ids=[])" ] }, { @@ -410,11 +1040,99 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Note: Feedback functions evaluated in the deferred manner can be seen in the \"Progress\" page of the TruLens dashboard." + "## Learn more about the call stack" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "json_like = last_record.layout_calls_as_app()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Munch({'record_id': 'record_hash_b5b6dc830a55b37fdffb2bbfa95df066', 'app_id': 'Chain1_ChatApplication_Filtered', 'cost': {'n_requests': 6, 'n_successful_requests': 6, 'n_classes': 0, 'n_tokens': 5171, 'n_stream_chunks': 0, 'n_prompt_tokens': 5102, 'n_completion_tokens': 69, 'cost': 0.007782000000000001}, 'perf': {'start_time': '2024-06-13T10:50:27.276380', 'end_time': '2024-06-13T10:50:31.586834'}, 'ts': '2024-06-13T10:50:31.586866', 'tags': '-', 'meta': None, 'main_input': 'What is Task Decomposition?', 'main_output': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'main_error': None, 'calls': [{'call_id': '6514d02b-7304-42bb-ab03-4b9cca2f35cf', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.question', 'method': {'obj': {'cls': {'name': 'RunnablePassthrough', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.passthrough'}, 'bases': None}, 'id': 13885082512, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13917059088, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': 'What is Task Decomposition?', 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.399239', 'end_time': '2024-06-13T10:50:27.420463'}, 'pid': 65698, 'tid': 6453375}, {'call_id': '87452040-fbc5-4522-a32d-1fcf7d317761', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'VectorStoreRetriever', 'module': {'package_name': 'langchain_core', 'module_name': 'langchain_core.vectorstores'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}], 'args': {'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, 'rets': [{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\", 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.442963', 'end_time': '2024-06-13T10:50:27.818347'}, 'pid': 65698, 'tid': 6453374}, {'call_id': 'c7f75d9a-e84c-4215-b4c3-a6c3497cc52a', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}, {'path': 'app.first.steps__.context.first.feedback', 'method': {'obj': {'cls': {'name': 'Feedback', 'module': {'package_name': 'trulens_eval.feedback', 'module_name': 'trulens_eval.feedback.feedback'}, 'bases': None}, 'id': 13885261616, 'init_bindings': None}, 'name': '__call__'}}], 'args': {'args': ['What is Task Decomposition?', \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\"]}, 'rets': 0.4, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.828037', 'end_time': '2024-06-13T10:50:28.276151'}, 'pid': 65698, 'tid': 6453383}, {'call_id': '4c6ed35c-e01b-4730-868b-9f72e5cdba02', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}, {'path': 'app.first.steps__.context.first.feedback', 'method': {'obj': {'cls': {'name': 'Feedback', 'module': {'package_name': 'trulens_eval.feedback', 'module_name': 'trulens_eval.feedback.feedback'}, 'bases': None}, 'id': 13885261616, 'init_bindings': None}, 'name': '__call__'}}], 'args': {'args': ['What is Task Decomposition?', 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.']}, 'rets': 0.4, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.828982', 'end_time': '2024-06-13T10:50:28.281506'}, 'pid': 65698, 'tid': 6453384}, {'call_id': '847a1d8e-4451-4fda-ba95-b799dbcc3dd0', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}, {'path': 'app.first.steps__.context.first.feedback', 'method': {'obj': {'cls': {'name': 'Feedback', 'module': {'package_name': 'trulens_eval.feedback', 'module_name': 'trulens_eval.feedback.feedback'}, 'bases': None}, 'id': 13885261616, 'init_bindings': None}, 'name': '__call__'}}], 'args': {'args': ['What is Task Decomposition?', 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.']}, 'rets': 0.7, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.822204', 'end_time': '2024-06-13T10:50:28.281915'}, 'pid': 65698, 'tid': 6453381}, {'call_id': 'e14d5a31-4785-457f-8c83-42d02d9552f1', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}, {'path': 'app.first.steps__.context.first.feedback', 'method': {'obj': {'cls': {'name': 'Feedback', 'module': {'package_name': 'trulens_eval.feedback', 'module_name': 'trulens_eval.feedback.feedback'}, 'bases': None}, 'id': 13885261616, 'init_bindings': None}, 'name': '__call__'}}], 'args': {'args': ['What is Task Decomposition?', 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.']}, 'rets': 0.2, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.825462', 'end_time': '2024-06-13T10:50:30.083574'}, 'pid': 65698, 'tid': 6453382}, {'call_id': 'e9a3cf02-7908-43b4-a5d7-cde30d4d82fd', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': '_get_relevant_documents'}}], 'args': {'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, 'rets': [{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.425547', 'end_time': '2024-06-13T10:50:30.084178'}, 'pid': 65698, 'tid': 6453374}, {'call_id': '229b7215-1669-45dd-91c2-0cc92410d191', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context.first', 'method': {'obj': {'cls': {'name': 'WithFeedbackFilterDocuments', 'module': {'package_name': 'trulens_eval.guardrails', 'module_name': 'trulens_eval.guardrails.langchain'}, 'bases': None}, 'id': 13914853328, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6415017872, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': [{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.402275', 'end_time': '2024-06-13T10:50:30.084741'}, 'pid': 65698, 'tid': 6453374}, {'call_id': 'cddb2f39-406a-43b3-abfb-beb4dec85349', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first.steps__.context', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914646800, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.383316', 'end_time': '2024-06-13T10:50:30.086065'}, 'pid': 65698, 'tid': 6453374}, {'call_id': 'cbc5fbc3-e7d9-4ef4-9dd0-1f7ff6affe8f', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.first', 'method': {'obj': {'cls': {'name': 'RunnableParallel', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 13886132880, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13871448208, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': {'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.336283', 'end_time': '2024-06-13T10:50:30.086599'}, 'pid': 65698, 'tid': 6452996}, {'call_id': '1cde13e5-7d19-4a35-9e65-410d94ff11b4', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.middle[0]', 'method': {'obj': {'cls': {'name': 'ChatPromptTemplate', 'module': {'package_name': 'langchain_core.prompts', 'module_name': 'langchain_core.prompts.chat'}, 'bases': None}, 'id': 13881883472, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': {'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6418097808, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': {'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:30.122172', 'end_time': '2024-06-13T10:50:30.152428'}, 'pid': 65698, 'tid': 6452996}, {'call_id': '469a1580-267e-4bcc-b8a1-f38b8fe0c04f', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.middle[1]', 'method': {'obj': {'cls': {'name': 'ChatOpenAI', 'module': {'package_name': 'langchain_community.chat_models', 'module_name': 'langchain_community.chat_models.openai'}, 'bases': None}, 'id': 13736902928, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': {'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946206736, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': {'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, 'error': None, 'perf': {'start_time': '2024-06-13T10:50:30.184183', 'end_time': '2024-06-13T10:50:31.530865'}, 'pid': 65698, 'tid': 6452996}, {'call_id': '4e9dee49-5efc-4ba4-8d37-c628a4d68cea', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}, {'path': 'app.last', 'method': {'obj': {'cls': {'name': 'StrOutputParser', 'module': {'package_name': 'langchain_core.output_parsers', 'module_name': 'langchain_core.output_parsers.string'}, 'bases': None}, 'id': 6414890448, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': {'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946133968, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, 'rets': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'error': None, 'perf': {'start_time': '2024-06-13T10:50:31.562307', 'end_time': '2024-06-13T10:50:31.586799'}, 'pid': 65698, 'tid': 6452996}, {'call_id': '3ad4449b-edbf-4ff3-9d61-87c6551fde9e', 'stack': [{'path': 'app', 'method': {'obj': {'cls': {'name': 'RunnableSequence', 'module': {'package_name': 'langchain_core.runnables', 'module_name': 'langchain_core.runnables.base'}, 'bases': None}, 'id': 6414884560, 'init_bindings': None}, 'name': 'invoke'}}], 'args': {'input': 'What is Task Decomposition?'}, 'rets': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'error': None, 'perf': {'start_time': '2024-06-13T10:50:27.276380', 'end_time': '2024-06-13T10:50:31.586834'}, 'pid': 65698, 'tid': 6452996}], 'app': {'first': {'steps__': {'question': {'invoke': [RecordAppCall(call_id='6514d02b-7304-42bb-ab03-4b9cca2f35cf', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.question, method=Method(obj=Obj(cls=langchain_core.runnables.passthrough.RunnablePassthrough, id=13885082512, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13917059088, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='What is Task Decomposition?', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 399239), end_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 420463)), pid=65698, tid=6453375)]}, 'context': {'first': {'_get_relevant_documents': [RecordAppCall(call_id='87452040-fbc5-4522-a32d-1fcf7d317761', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=langchain_core.vectorstores.VectorStoreRetriever, id=13914853328, init_bindings=None), name='_get_relevant_documents'))], args={'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\", 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}, {'page_content': 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 442963), end_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 818347)), pid=65698, tid=6453374), RecordAppCall(call_id='e9a3cf02-7908-43b4-a5d7-cde30d4d82fd', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents'))], args={'query': 'What is Task Decomposition?', 'run_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManagerForRetrieverRun', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914741136, 'init_bindings': None}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 425547), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 84178)), pid=65698, tid=6453374)], 'feedback': {'__call__': [RecordAppCall(call_id='c7f75d9a-e84c-4215-b4c3-a6c3497cc52a', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', \"(3) Task execution: Expert models execute on the specific tasks and log results.\\nInstruction:\\n\\nWith the input and the inference results, the AI assistant needs to describe the process and results. The previous stages can be formed as - User Input: {{ User Input }}, Task Planning: {{ Tasks }}, Model Selection: {{ Model Assignment }}, Task Execution: {{ Predictions }}. You must first answer the user's request in a straightforward manner. Then describe the task process and show your analysis and model inference results to the user in the first person. If inference results contain a file path, must tell the user the complete file path.\\n\\n(4) Response generation: LLM receives the execution results and provides summarized results to users.\\nTo put HuggingGPT into real world usage, a couple challenges need to solve: (1) Efficiency improvement is needed as both LLM inference rounds and interactions with other models slow down the process; (2) It relies on a long context window to communicate over complicated task content; (3) Stability improvement of LLM outputs and external model services.\\nAPI-Bank (Li et al. 2023) is a benchmark for evaluating the performance of tool-augmented LLMs. It contains 53 commonly used API tools, a complete tool-augmented LLM workflow, and 264 annotated dialogues that involve 568 API calls. The selection of APIs is quite diverse, including search engines, calculator, calendar queries, smart home control, schedule management, health data management, account authentication workflow and more. Because there are a large number of APIs, LLM first has access to API search engine to find the right API to call and then uses the corresponding documentation to make a call.\\n\\nFig. 12. Pseudo code of how LLM makes an API call in API-Bank. (Image source: Li et al. 2023)\\nIn the API-Bank workflow, LLMs need to make a couple of decisions and at each step we can evaluate how accurate that decision is. Decisions include:\\n\\nWhether an API call is needed.\\nIdentify the right API to call: if not good enough, LLMs need to iteratively modify the API inputs (e.g. deciding search keywords for Search Engine API).\\nResponse based on the API results: the model can choose to refine and call again if results are not satisfied.\\n\\nThis benchmark evaluates the agent’s tool use capabilities at three levels:\\n\\nLevel-1 evaluates the ability to call the API. Given an API’s description, the model needs to determine whether to call a given API, call it correctly, and respond properly to API returns.\\nLevel-2 examines the ability to retrieve the API. The model needs to search for possible APIs that may solve the user’s requirement and learn how to use them by reading documentation.\\nLevel-3 assesses the ability to plan API beyond retrieve and call. Given unclear user requests (e.g. schedule group meetings, book flight/hotel/restaurant for a trip), the model may have to conduct multiple API calls to solve it.\\n\\nCase Studies#\\nScientific Discovery Agent#\\nChemCrow (Bran et al. 2023) is a domain-specific example in which LLM is augmented with 13 expert-designed tools to accomplish tasks across organic synthesis, drug discovery, and materials design. The workflow, implemented in LangChain, reflects what was previously described in the ReAct and MRKLs and combines CoT reasoning with tools relevant to the tasks:\\n\\nThe LLM is provided with a list of tool names, descriptions of their utility, and details about the expected input/output.\\nIt is then instructed to answer a user-given prompt using the tools provided when necessary. The instruction suggests the model to follow the ReAct format - Thought, Action, Action Input, Observation.\"]}, rets=0.4, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 828037), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 276151)), pid=65698, tid=6453383), RecordAppCall(call_id='4c6ed35c-e01b-4730-868b-9f72e5cdba02', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 6. Illustration of how Algorithm Distillation (AD) works. (Image source: Laskin et al. 2023).\\nThe paper hypothesizes that any algorithm that generates a set of learning histories can be distilled into a neural network by performing behavioral cloning over actions. The history data is generated by a set of source policies, each trained for a specific task. At the training stage, during each RL run, a random task is sampled and a subsequence of multi-episode history is used for training, such that the learned policy is task-agnostic.\\nIn reality, the model has limited context window length, so episodes should be short enough to construct multi-episode history. Multi-episodic contexts of 2-4 episodes are necessary to learn a near-optimal in-context RL algorithm. The emergence of in-context RL requires long enough context.\\nIn comparison with three baselines, including ED (expert distillation, behavior cloning with expert trajectories instead of learning history), source policy (used for generating trajectories for distillation by UCB), RL^2 (Duan et al. 2017; used as upper bound since it needs online RL), AD demonstrates in-context RL with performance getting close to RL^2 despite only using offline RL and learns much faster than other baselines. When conditioned on partial training history of the source policy, AD also improves much faster than ED baseline.\\n\\nFig. 7. Comparison of AD, ED, source policy and RL^2 on environments that require memory and exploration. Only binary reward is assigned. The source policies are trained with A3C for \"dark\" environments and DQN for watermaze.(Image source: Laskin et al. 2023)\\nComponent Two: Memory#\\n(Big thank you to ChatGPT for helping me draft this section. I’ve learned a lot about the human brain and data structure for fast MIPS in my conversations with ChatGPT.)\\nTypes of Memory#\\nMemory can be defined as the processes used to acquire, store, retain, and later retrieve information. There are several types of memory in human brains.\\n\\n\\nSensory Memory: This is the earliest stage of memory, providing the ability to retain impressions of sensory information (visual, auditory, etc) after the original stimuli have ended. Sensory memory typically only lasts for up to a few seconds. Subcategories include iconic memory (visual), echoic memory (auditory), and haptic memory (touch).\\n\\n\\nShort-Term Memory (STM) or Working Memory: It stores information that we are currently aware of and needed to carry out complex cognitive tasks such as learning and reasoning. Short-term memory is believed to have the capacity of about 7 items (Miller 1956) and lasts for 20-30 seconds.\\n\\n\\nLong-Term Memory (LTM): Long-term memory can store information for a remarkably long time, ranging from a few days to decades, with an essentially unlimited storage capacity. There are two subtypes of LTM:\\n\\nExplicit / declarative memory: This is memory of facts and events, and refers to those memories that can be consciously recalled, including episodic memory (events and experiences) and semantic memory (facts and concepts).\\nImplicit / procedural memory: This type of memory is unconscious and involves skills and routines that are performed automatically, like riding a bike or typing on a keyboard.\\n\\n\\n\\n\\nFig. 8. Categorization of human memory.\\nWe can roughly consider the following mappings:\\n\\nSensory memory as learning embedding representations for raw inputs, including text, image or other modalities;\\nShort-term memory as in-context learning. It is short and finite, as it is restricted by the finite context window length of Transformer.\\nLong-term memory as the external vector store that the agent can attend to at query time, accessible via fast retrieval.']}, rets=0.4, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 828982), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 281506)), pid=65698, tid=6453384), RecordAppCall(call_id='847a1d8e-4451-4fda-ba95-b799dbcc3dd0', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.']}, rets=0.7, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 822204), end_time=datetime.datetime(2024, 6, 13, 10, 50, 28, 281915)), pid=65698, tid=6453381), RecordAppCall(call_id='e14d5a31-4785-457f-8c83-42d02d9552f1', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='_get_relevant_documents')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=13885261616, init_bindings=None), name='__call__'))], args={'args': ['What is Task Decomposition?', 'Fig. 10. A picture of a sea otter using rock to crack open a seashell, while floating in the water. While some other animals can use tools, the complexity is not comparable with humans. (Image source: Animals using tools)\\nMRKL (Karpas et al. 2022), short for β€œModular Reasoning, Knowledge and Language”, is a neuro-symbolic architecture for autonomous agents. A MRKL system is proposed to contain a collection of β€œexpert” modules and the general-purpose LLM works as a router to route inquiries to the best suitable expert module. These modules can be neural (e.g. deep learning models) or symbolic (e.g. math calculator, currency converter, weather API).\\nThey did an experiment on fine-tuning LLM to call a calculator, using arithmetic as a test case. Their experiments showed that it was harder to solve verbal math problems than explicitly stated math problems because LLMs (7B Jurassic1-large model) failed to extract the right arguments for the basic arithmetic reliably. The results highlight when the external symbolic tools can work reliably, knowing when to and how to use the tools are crucial, determined by the LLM capability.\\nBoth TALM (Tool Augmented Language Models; Parisi et al. 2022) and Toolformer (Schick et al. 2023) fine-tune a LM to learn to use external tool APIs. The dataset is expanded based on whether a newly added API call annotation can improve the quality of model outputs. See more details in the β€œExternal APIs” section of Prompt Engineering.\\nChatGPT Plugins and OpenAI API function calling are good examples of LLMs augmented with tool use capability working in practice. The collection of tool APIs can be provided by other developers (as in Plugins) or self-defined (as in function calls).\\nHuggingGPT (Shen et al. 2023) is a framework to use ChatGPT as the task planner to select models available in HuggingFace platform according to the model descriptions and summarize the response based on the execution results.\\n\\nFig. 11. Illustration of how HuggingGPT works. (Image source: Shen et al. 2023)\\nThe system comprises of 4 stages:\\n(1) Task planning: LLM works as the brain and parses the user requests into multiple tasks. There are four attributes associated with each task: task type, ID, dependencies, and arguments. They use few-shot examples to guide LLM to do task parsing and planning.\\nInstruction:\\n\\nThe AI assistant can parse user input to several tasks: [{\"task\": task, \"id\", task_id, \"dep\": dependency_task_ids, \"args\": {\"text\": text, \"image\": URL, \"audio\": URL, \"video\": URL}}]. The \"dep\" field denotes the id of the previous task which generates a new resource that the current task relies on. A special tag \"-task_id\" refers to the generated text image, audio and video in the dependency task with id as task_id. The task MUST be selected from the following options: {{ Available Task List }}. There is a logical relationship between tasks, please note their order. If the user input can\\'t be parsed, you need to reply empty JSON. Here are several cases for your reference: {{ Demonstrations }}. The chat history is recorded as {{ Chat History }}. From this chat history, you can find the path of the user-mentioned resources for your task planning.\\n\\n(2) Model selection: LLM distributes the tasks to expert models, where the request is framed as a multiple-choice question. LLM is presented with a list of models to choose from. Due to the limited context length, task type based filtration is needed.\\nInstruction:\\n\\nGiven the user request and the call command, the AI assistant helps the user to select a suitable model from a list of models to process the user request. The AI assistant merely outputs the model id of the most appropriate model. The output must be in a strict JSON format: \"id\": \"id\", \"reason\": \"your detail reason for the choice\". We have a list of models for you to choose from {{ Candidate Models }}. Please select one model from the list.']}, rets=0.2, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 825462), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 83574)), pid=65698, tid=6453382)]}, 'invoke': [RecordAppCall(call_id='229b7215-1669-45dd-91c2-0cc92410d191', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context.first, method=Method(obj=Obj(cls=trulens_eval.guardrails.langchain.WithFeedbackFilterDocuments, id=13914853328, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6415017872, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets=[{'page_content': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'metadata': {'source': 'https://lilianweng.github.io/posts/2023-06-23-agent/'}, 'type': 'Document'}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 402275), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 84741)), pid=65698, tid=6453374)]}, 'invoke': [RecordAppCall(call_id='cddb2f39-406a-43b3-abfb-beb4dec85349', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first.steps__.context, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13914646800, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 383316), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 86065)), pid=65698, tid=6453374)]}}, 'invoke': [RecordAppCall(call_id='cbc5fbc3-e7d9-4ef4-9dd0-1f7ff6affe8f', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.first, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableParallel, id=13886132880, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?', 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13871448208, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 336283), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 86599)), pid=65698, tid=6452996)]}, 'middle': [{'invoke': [RecordAppCall(call_id='1cde13e5-7d19-4a35-9e65-410d94ff11b4', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.middle[0], method=Method(obj=Obj(cls=langchain_core.prompts.chat.ChatPromptTemplate, id=13881883472, init_bindings=None), name='invoke'))], args={'input': {'context': 'Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results.', 'question': 'What is Task Decomposition?'}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 6418097808, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 122172), end_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 152428)), pid=65698, tid=6452996)]}, {'invoke': [RecordAppCall(call_id='469a1580-267e-4bcc-b8a1-f38b8fe0c04f', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.middle[1], method=Method(obj=Obj(cls=langchain_community.chat_models.openai.ChatOpenAI, id=13736902928, init_bindings=None), name='invoke'))], args={'input': {'messages': [{'content': 'You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don\\'t know the answer, just say that you don\\'t know. Use three sentences maximum and keep the answer concise.\\nQuestion: What is Task Decomposition? \\nContext: Fig. 1. Overview of a LLM-powered autonomous agent system.\\nComponent One: Planning#\\nA complicated task usually involves many steps. An agent needs to know what they are and plan ahead.\\nTask Decomposition#\\nChain of thought (CoT; Wei et al. 2022) has become a standard prompting technique for enhancing model performance on complex tasks. The model is instructed to β€œthink step by step” to utilize more test-time computation to decompose hard tasks into smaller and simpler steps. CoT transforms big tasks into multiple manageable tasks and shed lights into an interpretation of the model’s thinking process.\\nTree of Thoughts (Yao et al. 2023) extends CoT by exploring multiple reasoning possibilities at each step. It first decomposes the problem into multiple thought steps and generates multiple thoughts per step, creating a tree structure. The search process can be BFS (breadth-first search) or DFS (depth-first search) with each state evaluated by a classifier (via a prompt) or majority vote.\\nTask decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.\\nAnother quite distinct approach, LLM+P (Liu et al. 2023), involves relying on an external classical planner to do long-horizon planning. This approach utilizes the Planning Domain Definition Language (PDDL) as an intermediate interface to describe the planning problem. In this process, LLM (1) translates the problem into β€œProblem PDDL”, then (2) requests a classical planner to generate a PDDL plan based on an existing β€œDomain PDDL”, and finally (3) translates the PDDL plan back into natural language. Essentially, the planning step is outsourced to an external tool, assuming the availability of domain-specific PDDL and a suitable planner which is common in certain robotic setups but not in many other domains.\\nSelf-Reflection#\\nSelf-reflection is a vital aspect that allows autonomous agents to improve iteratively by refining past action decisions and correcting previous mistakes. It plays a crucial role in real-world tasks where trial and error are inevitable.\\nReAct (Yao et al. 2023) integrates reasoning and acting within LLM by extending the action space to be a combination of task-specific discrete actions and the language space. The former enables LLM to interact with the environment (e.g. use Wikipedia search API), while the latter prompting LLM to generate reasoning traces in natural language.\\nThe ReAct prompt template incorporates explicit steps for LLM to think, roughly formatted as:\\nThought: ...\\nAction: ...\\nObservation: ...\\n... (Repeated many times)\\n\\nFig. 2. Examples of reasoning trajectories for knowledge-intensive tasks (e.g. HotpotQA, FEVER) and decision-making tasks (e.g. AlfWorld Env, WebShop). (Image source: Yao et al. 2023).\\nIn both experiments on knowledge-intensive tasks and decision-making tasks, ReAct works better than the Act-only baseline where Thought: … step is removed.\\nReflexion (Shinn & Labash 2023) is a framework to equips agents with dynamic memory and self-reflection capabilities to improve reasoning skills. Reflexion has a standard RL setup, in which the reward model provides a simple binary reward and the action space follows the setup in ReAct where the task-specific action space is augmented with language to enable complex reasoning steps. After each action $a_t$, the agent computes a heuristic $h_t$ and optionally may decide to reset the environment to start a new trial depending on the self-reflection results. \\nAnswer:', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'human', 'name': None, 'id': None, 'example': False}]}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946206736, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets={'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 30, 184183), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 530865)), pid=65698, tid=6452996)]}], 'last': {'invoke': [RecordAppCall(call_id='4e9dee49-5efc-4ba4-8d37-c628a4d68cea', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke')), RecordAppCallMethod(path=Lens().app.last, method=Method(obj=Obj(cls=langchain_core.output_parsers.string.StrOutputParser, id=6414890448, init_bindings=None), name='invoke'))], args={'input': {'content': 'Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', 'additional_kwargs': {}, 'response_metadata': {'token_usage': {'completion_tokens': 65, 'prompt_tokens': 836, 'total_tokens': 901}, 'model_name': 'gpt-3.5-turbo', 'system_fingerprint': None, 'finish_reason': 'stop', 'logprobs': None}, 'type': 'ai', 'name': None, 'id': 'run-681f8f0e-131f-40f1-8690-0daba4978898-0', 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None}, 'config': {'tags': [], 'metadata': {}, 'callbacks': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'langchain_core.callbacks', 'module_name': 'langchain_core.callbacks.manager'}, 'bases': None}, 'id': 13946133968, 'init_bindings': None}}, 'recursion_limit': 25, 'configurable': {}}}, rets='Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 562307), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 586799)), pid=65698, tid=6452996)]}, 'invoke': [RecordAppCall(call_id='3ad4449b-edbf-4ff3-9d61-87c6551fde9e', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=langchain_core.runnables.base.RunnableSequence, id=6414884560, init_bindings=None), name='invoke'))], args={'input': 'What is Task Decomposition?'}, rets='Task decomposition is a technique used to break down complex tasks into smaller and simpler steps. It can be achieved through prompting techniques like Chain of Thought or Tree of Thoughts, or by using task-specific instructions. Task decomposition can also involve outsourcing the planning step to an external classical planner, as seen in the LLM+P approach.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 13, 10, 50, 27, 276380), end_time=datetime.datetime(2024, 6, 13, 10, 50, 31, 586834)), pid=65698, tid=6452996)]}})" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_like" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "92d5f72759144d6f8ac46e39f7387841", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Tree(nodes=(Node(name='Record ID: record_hash_b5b6dc830a55b37fdffb2bbfa95df066'), Node(name='App ID: Chain1_Ch…" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from ipytree import Tree, Node\n", + "\n", + "def display_call_stack(data):\n", + " tree = Tree()\n", + " tree.add_node(Node('Record ID: {}'.format(data['record_id'])))\n", + " tree.add_node(Node('App ID: {}'.format(data['app_id'])))\n", + " tree.add_node(Node('Cost: {}'.format(data['cost'])))\n", + " tree.add_node(Node('Performance: {}'.format(data['perf'])))\n", + " tree.add_node(Node('Timestamp: {}'.format(data['ts'])))\n", + " tree.add_node(Node('Tags: {}'.format(data['tags'])))\n", + " tree.add_node(Node('Main Input: {}'.format(data['main_input'])))\n", + " tree.add_node(Node('Main Output: {}'.format(data['main_output'])))\n", + " tree.add_node(Node('Main Error: {}'.format(data['main_error'])))\n", + " \n", + " calls_node = Node('Calls')\n", + " tree.add_node(calls_node)\n", + " \n", + " for call in data['calls']:\n", + " call_node = Node('Call')\n", + " calls_node.add_node(call_node)\n", + " \n", + " for step in call['stack']:\n", + " step_node = Node('Step: {}'.format(step['path']))\n", + " call_node.add_node(step_node)\n", + " if 'expanded' in step:\n", + " expanded_node = Node('Expanded')\n", + " step_node.add_node(expanded_node)\n", + " for expanded_step in step['expanded']:\n", + " expanded_step_node = Node('Step: {}'.format(expanded_step['path']))\n", + " expanded_node.add_node(expanded_step_node)\n", + " \n", + " return tree\n", + "\n", + "# Usage\n", + "tree = display_call_stack(json_like)\n", + "tree" ] } ], @@ -434,7 +1152,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.11.9" }, "vscode": { "interpreter": { diff --git a/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb b/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb index 82952024c..1f88ffac8 100644 --- a/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb +++ b/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb @@ -9,7 +9,9 @@ "\n", "In this quickstart you will create a simple Llama Index app and learn how to log it and get feedback on an LLM response.\n", "\n", - "For evaluation, we will leverage the \"hallucination triad\" of groundedness, context relevance and answer relevance.\n", + "You'll also learn how to use feedbacks for guardrails, via filtering retrieved context.\n", + "\n", + "For evaluation, we will leverage the RAG triad of groundedness, context relevance and answer relevance.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb)" ] @@ -27,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -40,12 +42,12 @@ "metadata": {}, "source": [ "### Add API keys\n", - "For this quickstart, you will need Open AI and Huggingface keys. The OpenAI key is used for embeddings and GPT, and the Huggingface key is used for evaluation." + "For this quickstart, you will need an Open AI key. The OpenAI key is used for embeddings, completion and evaluation." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -63,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -84,11 +86,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "!wget https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt -P data/" + "import os\n", + "if not os.path.exists('data/paul_graham_essay.txt'):\n", + " !wget https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt -P data/\n" ] }, { @@ -103,16 +107,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n", + "from llama_index.core import Settings\n", + "from llama_index.llms.openai import OpenAI\n", + "\n", + "Settings.chunk_size = 128\n", + "Settings.chunk_overlap = 16\n", + "Settings.llm = OpenAI()\n", "\n", "documents = SimpleDirectoryReader(\"data\").load_data()\n", "index = VectorStoreIndex.from_documents(documents)\n", "\n", - "query_engine = index.as_query_engine()" + "query_engine = index.as_query_engine(similarity_top_k=3)" ] }, { @@ -125,9 +135,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The author worked on writing and programming outside of school before college.\n" + ] + } + ], "source": [ "response = query_engine.query(\"What did the author do growing up?\")\n", "print(response)" @@ -143,9 +161,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In groundedness_measure_with_cot_reasons, input source will be set to __record__.app.query.rets.source_nodes[:].node.text.collect() .\n", + "βœ… In groundedness_measure_with_cot_reasons, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In context_relevance_with_cot_reasons, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In context_relevance_with_cot_reasons, input context will be set to __record__.app.query.rets.source_nodes[:].node.text .\n" + ] + } + ], "source": [ "from trulens_eval.feedback.provider import OpenAI\n", "from trulens_eval import Feedback\n", @@ -189,7 +220,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -201,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -210,6 +241,236 @@ " query_engine.query(\"What did the author do growing up?\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Use guardrails\n", + "\n", + "In addition to making informed iteration, we can also directly use feedback results as guardrails at inference time. In particular, here we show how to use the context relevance score as a guardrail to filter out irrelevant context before it gets passed to the LLM. This both reduces hallucination and improves efficiency.\n", + "\n", + "Below, you can see the TruLens feedback display of each context relevance chunk retrieved by our RAG." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0What did the author do growing up?I remember taking the boys to the coast on a s...0.2
1What did the author do growing up?What I Worked On\\n\\nFebruary 2021\\n\\nBefore co...0.8
2What did the author do growing up?Idelle was in New York at least, and there wer...0.2
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 What did the author do growing up? \n", + "1 What did the author do growing up? \n", + "2 What did the author do growing up? \n", + "\n", + " context ret \n", + "0 I remember taking the boys to the coast on a s... 0.2 \n", + "1 What I Worked On\\n\\nFebruary 2021\\n\\nBefore co... 0.8 \n", + "2 Idelle was in New York at least, and there wer... 0.2 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "last_record = recording.records[-1]\n", + "\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'context_relevance_with_cot_reasons')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Wouldn't it be great if we could automatically filter out context chunks with relevance scores below 0.5?\n", + "\n", + "We can do so with the TruLens guardrail, *WithFeedbackFilterNodes*. All we have to do is use the method `of_query_engine` to create a new filtered retriever, passing in the original retriever along with the feedback function and threshold we want to use." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In context_relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In context_relevance, input context will be set to __record__.app.query.rets.source_nodes[:].node.text .\n" + ] + } + ], + "source": [ + "from trulens_eval.guardrails.llama import WithFeedbackFilterNodes\n", + "\n", + "# note: feedback function used for guardrail must only return a score, not also reasons\n", + "f_context_relevance_score = (\n", + " Feedback(provider.context_relevance)\n", + " .on_input()\n", + " .on(context)\n", + " .aggregate(np.mean)\n", + ")\n", + "\n", + "filtered_query_engine = WithFeedbackFilterNodes(query_engine, feedback=f_context_relevance_score, threshold=0.5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we can operate as normal" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Response(response='The author worked on writing and programming outside of school before college.', source_nodes=[NodeWithScore(node=TextNode(id_='3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', embedding=None, metadata={'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, excluded_embed_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], excluded_llm_metadata_keys=['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], relationships={: RelatedNodeInfo(node_id='7ca0bef3-9d52-437d-86b9-9e48eae8e467', node_type=, metadata={'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, hash='55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'), : RelatedNodeInfo(node_id='3976e410-ba26-4cc0-bc98-0475d9f33c1b', node_type=, metadata={}, hash='e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed')}, text=\"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", start_char_idx=2, end_char_idx=373, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.8207477152744018)], metadata={'3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}})" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "tru_recorder = TruLlama(filtered_query_engine,\n", + " app_id='LlamaIndex_App1_Filtered',\n", + " feedbacks=[f_answer_relevance, f_context_relevance, f_groundedness])\n", + "\n", + "with tru_recorder as recording:\n", + " llm_response = filtered_query_engine.query(\"What did the author do growing up?\")\n", + "\n", + "display(llm_response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## See the power of context filters!\n", + "\n", + "If we inspect the context relevance of our retreival now, you see only relevant context chunks!" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0What did the author do growing up?What I Worked On\\n\\nFebruary 2021\\n\\nBefore co...0.8
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 What did the author do growing up? \n", + "\n", + " context ret \n", + "0 What I Worked On\\n\\nFebruary 2021\\n\\nBefore co... 0.8 " + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "last_record = recording.records[-1]\n", + "\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'context_relevance_with_cot_reasons')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -219,9 +480,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Record(record_id='record_hash_404429446b4fc7f465daf0980c6fedc9', app_id='LlamaIndex_App1_Filtered', cost=Cost(n_requests=5, n_successful_requests=5, n_classes=0, n_tokens=1322, n_stream_chunks=0, n_prompt_tokens=1306, n_completion_tokens=16, cost=0.0019790000000000003), perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 49, 921442), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 607337)), ts=datetime.datetime(2024, 6, 10, 19, 53, 51, 607644), tags='-', meta=None, main_input='What did the author do growing up?', main_output='The author worked on writing and programming outside of school before college.', main_error=None, calls=[RecordAppCall(call_id='1cb6e4a3-c3d1-4a2a-905c-088956e59902', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='retrieve')), RecordAppCallMethod(path=Lens().app.query_engine._retriever, method=Method(obj=Obj(cls=llama_index.core.indices.vector_store.retrievers.retriever.VectorIndexRetriever, id=14240242384, init_bindings=None), name='retrieve')), RecordAppCallMethod(path=Lens().app.query_engine._retriever, method=Method(obj=Obj(cls=llama_index.core.indices.vector_store.retrievers.retriever.VectorIndexRetriever, id=14240242384, init_bindings=None), name='_retrieve'))], args={'query_bundle': {'query_str': 'What did the author do growing up?', 'image_path': None, 'custom_embedding_strs': None, 'embedding': [0.012144206091761589, -0.015698930248618126, 0.007664461154490709, -0.010062908753752708, -0.021275486797094345, 0.02139441855251789, -0.006689885165542364, -0.01778683438897133, -0.027882780879735947, -0.03047283925116062, 0.024301627650856972, 0.0009704462718218565, 0.0025801481679081917, 0.01097471546381712, 0.003690173616632819, 0.01857971027493477, 0.03898303583264351, -0.009613612666726112, -0.0009795313235372305, -0.019055435433983803, -0.0023604556918144226, 0.010723638348281384, 0.0035844568628817797, -0.008569660596549511, -0.003151679178699851, 0.006812119856476784, 0.025041643530130386, -0.028543509542942047, 0.023363390937447548, -0.0033779789227992296, 0.011450440622866154, -0.013901746831834316, -0.008450728841125965, -0.005563341546803713, -0.024883069097995758, -0.009395572356879711, -0.016082152724266052, -0.014469973742961884, 0.0017724066274240613, 0.00755874440073967, 0.016095368191599846, -0.0030955171678215265, -0.013888532295823097, -0.0018087467178702354, 0.02201550267636776, 0.007327489089220762, -0.010360237210988998, -0.0012966814683750272, -0.016068939119577408, -0.0013982686214148998, 0.03425221145153046, 0.013406199403107166, -0.004350902978330851, -0.024103408679366112, -0.019491517916321754, 0.0012314344057813287, 0.0026974277570843697, 0.014099964872002602, 0.0016691676573827863, -0.021275486797094345, -0.007023553363978863, 0.022174078971147537, -0.00299640791490674, 0.007413383573293686, -0.024750923737883568, 0.0007007861277088523, -0.013194765895605087, -0.005199940409511328, -0.008424299769103527, -0.0153157077729702, 0.035996537655591965, 0.00011222076136618853, -0.009904333390295506, -0.0027948853094130754, 0.019002577289938927, -0.010941678658127785, -0.00964004173874855, -0.007367132697254419, -0.009957191534340382, 0.010611314326524734, -0.00785607285797596, -0.0219230018556118, -0.0064322007820010185, 0.009560754522681236, 0.01099453680217266, -0.005642629228532314, 0.016808954998850822, 0.029627105221152306, -0.020257962867617607, -0.01844756491482258, -0.0026214439421892166, 0.017601830884814262, 0.01657109335064888, 0.015130703337490559, 0.014773909002542496, 0.0032887805718928576, -0.01564607210457325, 0.01129186525940895, -0.008840559050440788, -0.003425881965085864, -0.005804507527500391, 0.0041592917405068874, -0.007195343263447285, -0.012401890940964222, -0.01877792924642563, 0.013036190532147884, -0.0062967510893940926, -0.021909786388278008, 0.028305647894740105, -0.03242859989404678, -0.014086750335991383, 0.0202711783349514, 0.02759205922484398, -0.05277906358242035, -0.008543231524527073, -0.025226648896932602, 0.0007338225841522217, 0.000338417332386598, 0.008952883072197437, 0.0009630130953155458, 0.007968396879732609, 0.001654301187954843, 0.021962644532322884, -0.010399880819022655, 0.00944182276725769, -0.0012454749085009098, -0.015897149220108986, -0.00881412997841835, -0.011126683093607426, -0.007102841045707464, 0.017469685524702072, -0.0016848599771037698, 0.012481178157031536, -0.013478879816830158, -0.017667904496192932, 0.016610736027359962, -0.019029006361961365, 0.015342136844992638, -0.021473705768585205, -0.02525307796895504, 0.020482610911130905, 0.022914094850420952, -0.008179829455912113, 0.002928683068603277, 0.006983909755945206, 0.018473993986845016, 0.030552126467227936, 0.039062321186065674, 0.01008273009210825, -0.018698642030358315, 0.017469685524702072, -0.0025884073693305254, 0.007089626509696245, -0.008457336574792862, 0.01437747199088335, 0.035071514546871185, -0.015381780453026295, 0.012540644034743309, -0.016002865508198738, -0.01994081400334835, 0.0209319069981575, 0.025689158588647842, 0.0221476498991251, -0.008212866261601448, -0.01170151773840189, 0.03816372901201248, 0.023416249081492424, 0.014958913438022137, 0.0028097517788410187, -0.012679397128522396, -0.024050550535321236, 0.012593502178788185, -0.03351219370961189, 0.009435215964913368, 0.007281237747520208, 0.024235554039478302, -0.0026924721896648407, 0.015540354885160923, -0.01706003211438656, -0.003908214159309864, -0.012666182592511177, 0.001992099219933152, 0.02501521445810795, 0.04424244165420532, -0.008622518740594387, -0.00881412997841835, 0.00861591100692749, 0.008080720901489258, -0.004327777307480574, 0.0031896710861474276, -0.012124384753406048, 0.024473417550325394, -0.012501000426709652, -0.02449984662234783, -0.66982102394104, -0.012752077542245388, -0.0032127967569977045, -0.01967652142047882, 0.01149008423089981, 0.022755520418286324, -0.002528941724449396, 0.0365779809653759, -0.0306314155459404, 0.004301348235458136, -0.007320881821215153, 0.03858659788966179, 0.026719896122813225, 0.0004080004000570625, 0.011866699904203415, -0.011476869694888592, 0.010109160095453262, -0.02218729257583618, 0.00853001605719328, -0.015342136844992638, 0.00508431252092123, 0.02073368802666664, -0.007624817080795765, -0.0062571074813604355, 0.008576267398893833, 0.010743459686636925, 0.008992526680231094, -0.03311575576663017, 0.013835673220455647, 0.02571558766067028, -0.017839694395661354, 0.028807802125811577, 0.03647226095199585, 0.025794874876737595, 0.05513126030564308, -0.012322602793574333, -0.008378048427402973, 0.03536223620176315, 0.005566644947975874, 0.04387243092060089, -0.02062797173857689, -0.016676809638738632, 0.015619643032550812, 0.014205682091414928, 0.007710712030529976, -0.01119275577366352, 0.039564475417137146, 0.0032887805718928576, 0.035071514546871185, 0.019266869872808456, -0.005854062270373106, -0.001008438179269433, -0.001815353985875845, -0.007922145538032055, -0.0007602517725899816, -0.006504880730062723, 0.0012182198697701097, 0.015421424061059952, 0.007294452283531427, 0.017800049856305122, -0.006389253307133913, 0.03715942054986954, -0.023033026605844498, 0.004704393446445465, -0.0033829344902187586, 0.02241194061934948, -0.016544664278626442, 0.026204528287053108, -0.005606289021670818, -0.022636588662862778, 0.009778794832527637, 0.007585173472762108, -0.015289277769625187, -0.009474859572947025, 0.01335334125906229, 0.010538633912801743, 0.03274574875831604, 0.005295746028423309, -0.002448002342134714, -0.006716314237564802, -0.027023831382393837, -0.0037000845186412334, -0.04067450016736984, -0.010849176906049252, 0.02205514721572399, 0.006448718719184399, -0.03023497760295868, 0.007922145538032055, -0.003838837845250964, 0.007677675690501928, -0.01026773452758789, 0.03277217969298363, -0.007690890226513147, -0.026019522920250893, -0.012540644034743309, 0.009811831638216972, -0.013445843011140823, -0.001188487047329545, 0.0007317577837966383, 0.002581800101324916, 0.010611314326524734, -0.002264650072902441, 0.010221484117209911, 0.011338116601109505, 0.00550387566909194, 0.000641320482827723, 0.012434926815330982, 0.025530584156513214, 0.03927375376224518, -0.018923290073871613, 0.005708701908588409, 0.0009473207755945623, 0.0013354993425309658, -0.002353848423808813, -0.007869287393987179, -0.023151958361268044, 0.014033892191946507, 0.008292153477668762, 0.011001144535839558, -0.013459057547152042, -0.0014023981057107449, 0.010413095355033875, -0.00911145843565464, -0.018209701403975487, -0.005014935974031687, 0.023455893620848656, -0.009197353385388851, 0.010703816078603268, -0.011357937939465046, 0.0007561221718788147, -0.0007342355675064027, -0.010023265145719051, 0.013300483115017414, -0.01910829357802868, 0.013769600540399551, 0.0031285537406802177, -0.003805801272392273, -0.0061943382024765015, 0.00477707339450717, -0.00725480867549777, -0.020509039983153343, 0.004998418036848307, -0.005209851078689098, -0.003214448457583785, -0.017152534797787666, -0.02046939730644226, 0.01117293443530798, -0.002784974407404661, -0.012765292078256607, 0.02152656391263008, 0.003366416320204735, -0.013518523424863815, -0.022517656907439232, -0.015883933752775192, -0.01828899048268795, -0.016201084479689598, 0.01265957485884428, -0.005351908039301634, -0.01431139837950468, 0.009085029363632202, -0.0023109009489417076, 0.03628725931048393, -0.010928464122116566, 0.010155410505831242, -0.014218896627426147, 0.01036684401333332, 0.013181551359593868, 0.012296173721551895, 0.010868998244404793, -0.008701805956661701, 0.011272042989730835, -0.016148226335644722, 0.0034490074031054974, -0.005170207470655441, -0.007446420378983021, 0.012652968056499958, 0.0008837255882099271, -0.002722205128520727, -0.008477157913148403, 0.004420279525220394, -0.004479745402932167, -0.007902323268353939, -0.007882501929998398, -0.011575979180634022, 0.023706970736384392, -0.009382357820868492, 0.0012628190452232957, 0.03245502710342407, 0.016201084479689598, 0.03227002173662186, 7.371262472588569e-05, 0.014536046423017979, 0.006762565113604069, 0.009058600291609764, -0.0006429722998291254, -0.009461645036935806, 0.030049972236156464, 0.007049982436001301, 0.009930762462317944, 0.00785607285797596, -0.026191312819719315, 0.026191312819719315, 0.01399424858391285, 0.0019755808170884848, 0.017245037481188774, -0.037317994982004166, -0.004004020243883133, -0.02023153379559517, -0.008219473995268345, 0.0025223344564437866, 0.003528294852003455, -0.008153400383889675, 0.006025852169841528, -0.018989363685250282, -0.01437747199088335, 0.006141479592770338, -0.018077556043863297, 0.017152534797787666, -0.01047916803508997, 0.001551062217913568, 0.017932195216417313, -0.011873307637870312, -0.004704393446445465, -0.001744325621984899, -0.015223205089569092, 0.019861524924635887, 0.001973929116502404, -0.019385799765586853, 0.006937658414244652, -0.022068360820412636, -0.004836539272218943, -0.004796895198523998, 0.01090864185243845, 0.01802469789981842, -0.005242887884378433, 0.021143341436982155, 0.02218729257583618, -0.01544785313308239, 0.01835506223142147, -0.011675088666379452, -0.00881412997841835, 0.016861815005540848, 0.024737708270549774, -0.005655843764543533, 0.030552126467227936, -0.011959201656281948, 0.017773620784282684, -0.021169770509004593, -0.011893128976225853, -0.0057483455166220665, 0.003977591171860695, -0.012084740214049816, 0.0026743023190647364, -0.023680541664361954, 0.006250500213354826, 0.007895716466009617, 0.005355211906135082, -0.004192328080534935, 0.00868198461830616, 0.02957424707710743, -0.0042352755554020405, 0.009626827202737331, -0.003957768902182579, 0.01706003211438656, 0.0014511268818750978, 0.013703527860343456, -0.009659864008426666, -0.005081009119749069, -0.03219073638319969, -0.0105782775208354, -0.022755520418286324, -0.010644350200891495, 0.006716314237564802, -0.02126227132976055, 0.010743459686636925, 0.007307667285203934, -0.00924360379576683, -0.004988506902009249, 0.00911145843565464, -0.0010546892881393433, -0.02261015959084034, -0.0025702372658997774, 0.022821594029664993, 0.01716575026512146, 0.0043608141131699085, -0.007003731559962034, -0.007076411973685026, 0.0016361312009394169, -0.014033892191946507, -0.0004860490735154599, 0.005262709688395262, -0.002244828036054969, 0.0026528285816311836, -0.010849176906049252, 0.0041592917405068874, 0.016465377062559128, 0.03020854853093624, -0.023588038980960846, 0.023521967232227325, -0.02099798060953617, -0.03345933556556702, -0.009177531115710735, -0.022359082475304604, 0.0017509328899905086, 0.02838493511080742, 0.005150385666638613, -0.014641763642430305, -0.013095656409859657, -0.007003731559962034, -0.007023553363978863, -0.012302781455218792, 0.0009704462718218565, 0.0061447834596037865, 0.01338637713342905, 0.010360237210988998, -0.011278650723397732, -0.014694621786475182, -0.006039066705852747, 0.015936793759465218, 0.0032028856221586466, -0.026415960863232613, -0.021275486797094345, -0.029124950990080833, 0.012104562483727932, 0.1068795844912529, 0.014403901062905788, -0.01564607210457325, 0.015077845193445683, 0.009785402566194534, -0.011424011550843716, -0.016161441802978516, -0.021804070100188255, 0.01004969421774149, -0.0018682123627513647, -0.00121904572006315, -0.011873307637870312, 0.01411317940801382, -0.013716742396354675, -0.007076411973685026, -0.0016295238165184855, -3.1307215976994485e-05, -0.025755232200026512, -0.01523641962558031, -0.021315129473805428, 0.003528294852003455, 0.026230957359075546, 0.010095945559442043, 0.022636588662862778, 0.007809821516275406, -0.006267018150538206, 0.004080004058778286, 0.0010505596874281764, 0.0286492258310318, -0.01076328195631504, 0.00599281582981348, 0.0006190208368934691, 0.009461645036935806, -0.015513925813138485, 0.006415682379156351, 0.04109736904501915, 0.014086750335991383, 0.0287549439817667, 0.02525307796895504, -0.0040106275118887424, 0.031635724008083344, -0.009428608231246471, 0.014205682091414928, -0.0352565199136734, 0.014060321263968945, -0.026363102719187737, 0.0026974277570843697, 0.028173500671982765, 0.0040866113267838955, 0.0007334096007980406, 0.0024314841721206903, -0.0022894274443387985, -0.0057483455166220665, -0.007684282958507538, 0.014998557046055794, -0.00549726840108633, 0.013848887756466866, -0.0032161003910005093, -0.0018748196307569742, 0.02627060003578663, -0.010849176906049252, -0.031609293073415756, 0.009283248335123062, -0.026257386431097984, 0.007301060017198324, -0.02719562128186226, -0.013207980431616306, 0.0042716157622635365, -0.009501288644969463, -0.002657783916220069, -0.009236996993422508, -0.00550387566909194, -0.038110870867967606, -0.015831075608730316, 0.024024121463298798, 0.01835506223142147, 0.039564475417137146, -0.004829932004213333, -0.008424299769103527, 0.00785607285797596, -0.0031615900807082653, -0.02994425594806671, 0.018500423058867455, -0.018764715641736984, 0.00019966416584793478, 0.024830210953950882, -0.0059201354160904884, -0.01450961735099554, -0.020614756271243095, 0.008741449564695358, 0.0012396934907883406, 0.00745963491499424, -0.004582158289849758, -0.04244525730609894, -0.012798327952623367, -0.023218030110001564, -0.010670779272913933, 0.01257367990911007, 0.01641251891851425, -0.014443544670939445, 0.0061943382024765015, -0.023257674649357796, -0.028702083975076675, -0.006554435472935438, -0.0042484900914132595, -0.0007796607096679509, -0.005758256651461124, 0.019293298944830894, -0.022914094850420952, -0.02851708047091961, 0.03766157478094101, -0.006554435472935438, 0.0002335265453439206, 0.017588617280125618, 0.015791432932019234, 0.0345957912504673, -0.0055104829370975494, 0.008113756775856018, 0.0038586596492677927, -0.011324902065098286, -0.002576844533905387, -0.02644238993525505, 0.020680829882621765, 0.018460778519511223, -0.037873007357120514, -0.009085029363632202, -0.008060898631811142, -0.02225336618721485, -0.018870431929826736, -0.005137171130627394, 0.0004455793823581189, 0.02745991386473179, -0.031239286065101624, -0.013769600540399551, -0.012157420627772808, 0.00023311359109357, -0.007935360074043274, -5.884621259610867e-06, 0.002801492577418685, 0.0002467411395628005, -0.023482322692871094, -0.01690145768225193, -0.005612896289676428, -0.018619354814291, 0.002233265433460474, -0.02706347592175007, -0.003321816911920905, -0.006931051146239042, -0.012712433934211731, 0.03874517232179642, -0.008477157913148403, 0.0021110305096954107, 0.010003442876040936, 0.017918981611728668, -0.026521677151322365, -0.02571558766067028, -0.008721628226339817, 0.020125817507505417, 0.04067450016736984, -0.0031467238441109657, 0.019332941621541977, -0.00912467297166586, 0.03462221845984459, 0.008259117603302002, -0.00028266830486245453, 0.01584429107606411, 0.0076578538864851, 0.0031037763692438602, -0.005315567832440138, 0.03895660489797592, 0.014932484365999699, 0.0211565550416708, -0.0019425443606451154, 0.0017311109695583582, 0.007135877385735512, 0.024843424558639526, 0.0022084880620241165, -0.004228668287396431, -9.100515308091417e-05, -0.03557366877794266, 0.005692183505743742, 0.014218896627426147, -0.007109448313713074, 0.011252221651375294, 0.008622518740594387, 0.002487646182999015, 0.03425221145153046, 0.005391551647335291, 0.010545240715146065, -0.02267623320221901, 0.02231943979859352, -0.015302492305636406, 0.03382934629917145, -0.018170058727264404, -0.005325478967279196, -0.014575690031051636, -0.017311109229922295, -0.02119619958102703, 0.012421712279319763, 0.02161906659603119, 0.007836250588297844, 0.008358227089047432, -0.00020957511151209474, -0.013558167032897472, -0.00010520051000639796, -0.016082152724266052, 0.008371441625058651, 9.972884436137974e-05, 0.015209990553557873, 0.005536912474781275, 0.018196487799286842, -0.04149380698800087, -0.007505885791033506, -0.01712610572576523, 0.0013462360948324203, -0.01857971027493477, -0.018170058727264404, 0.023786257952451706, 0.0035745459608733654, -0.00955414678901434, -0.022914094850420952, -0.010750067420303822, 0.030974993482232094, -0.005814418662339449, 0.002353848423808813, -0.01875150017440319, -0.02194943092763424, -0.005705398507416248, 0.010717030614614487, 0.002978237811475992, -0.01326744630932808, 0.012692611664533615, 0.011794019490480423, 0.01437747199088335, -0.006904622074216604, 0.005127259995788336, 0.018341848626732826, -0.026455605402588844, -0.01613501086831093, -0.00467796390876174, 0.006508184596896172, -0.025966664776206017, 0.012395283207297325, -0.008219473995268345, -0.021605851128697395, 0.02666703797876835, -0.005768167786300182, -0.008543231524527073, -0.008800915442407131, -0.017945410683751106, -0.016068939119577408, 0.0101355891674757, 0.01584429107606411, 0.00632978742942214, -0.01809077151119709, 0.01089542731642723, -0.005470839329063892, -0.029257098212838173, 0.015355351381003857, 0.014258540235459805, -0.013571381568908691, 0.0068319421261549, 0.0018533458933234215, 0.02013903111219406, 0.0037661574315279722, -0.0012809891486540437, 0.0025355489924550056, -0.017733976244926453, -0.024209124967455864, 0.026402747258543968, 0.00548075046390295, -0.005021543242037296, -0.02036367915570736, 0.01242832001298666, 0.0077701774425804615, 0.026521677151322365, -0.018923290073871613, -0.009884512051939964, 0.016690025106072426, -0.009303069673478603, 0.010888820514082909, 0.004321170039474964, -0.03562653064727783, 0.009098243899643421, -0.009382357820868492, -0.018302204087376595, -0.014892840757966042, -0.011457047425210476, 0.012884222902357578, -0.031609293073415756, -0.04247168451547623, -0.0033251207787543535, 0.011014359071850777, 0.012553858570754528, -0.03647226095199585, -0.003501865779981017, 0.015738574787974358, 0.0273277685046196, -0.029759252443909645, 0.012606716714799404, -0.007228379603475332, -0.013412807136774063, -0.007677675690501928, 0.005880491808056831, -0.012699219398200512, -0.008444122038781643, 0.039035893976688385, -0.0013478879118338227, -0.025160575285553932, -0.009316284209489822, -0.0010183491976931691, -0.013201373629271984, -0.022570516914129257, -0.009580575861036777, -0.023521967232227325, 0.008933061733841896, -0.008767878636717796, -0.02849065139889717, -0.008193044923245907, 0.010188447311520576, 0.004208846017718315, -0.004070092923939228, 0.004790287930518389, -0.004780377261340618, -0.004453316330909729, 0.0062934476882219315, -0.003538205986842513, -0.0028543509542942047, -0.024975571781396866, 0.004783680662512779, 0.0001777775032678619, 0.014575690031051636, -0.022861236706376076, 0.00725480867549777, -0.02069404534995556, 0.002936942270025611, -0.0015444549499079585, 0.03364434093236923, 0.021975859999656677, 0.0025124235544353724, -0.003954465501010418, 0.028702083975076675, 0.005563341546803713, 0.012454749085009098, 0.01544785313308239, -0.009858082979917526, -0.0027106422930955887, 0.000394372851587832, -0.04323813319206238, -0.01038005854934454, 0.016346445307135582, 0.036392975598573685, 0.009904333390295506, -0.01277189888060093, -0.01957080513238907, -0.012296173721551895, -0.03821658715605736, -0.00755213713273406, 0.00736052542924881, 0.0006103487685322762, 0.017271466553211212, -0.008510194718837738, -0.007043375167995691, 0.007915537804365158, -0.017271466553211212, -0.0021870143245905638, -0.0005079357069917023, -0.010802925564348698, -0.020416539162397385, 0.01014880370348692, 0.01877792924642563, -0.001067077973857522, -0.0005087616154924035, -0.004076700191944838, 0.0007639683899469674, -0.011675088666379452, -0.002487646182999015, -0.02304624207317829, -0.01034702267497778, -0.002573540899902582, -0.009633434936404228, 0.03575867414474487, 0.013009761460125446, 0.00035803273203782737, 0.004268311895430088, -0.0286492258310318, -0.00962022040039301, -0.0018748196307569742, -0.020654400810599327, -0.0026511766482144594, -0.010062908753752708, 0.016623951494693756, -0.004195631481707096, 0.008675376884639263, 0.008232688531279564, -0.014932484365999699, 0.0153157077729702, 0.014284969307482243, 0.027116334065794945, -0.020773332566022873, -0.01881757378578186, 0.025134146213531494, -0.00631326949223876, -0.01242832001298666, -0.022663017734885216, 0.021711567416787148, -0.00233072298578918, -0.0020135727245360613, -0.014456759206950665, -0.03597010672092438, 0.014747479930520058, 0.00012626126408576965, -0.004995114170014858, -0.006785690784454346, 0.004674660507589579, -0.0038322305772453547, -0.0015279367798939347, 0.011113468557596207, 0.012593502178788185, -0.0002702796191442758, -0.019266869872808456, 0.006729528773576021, 0.013901746831834316, -0.006138176191598177, -0.00716230645775795, -0.0254512969404459, 0.02538522332906723, -0.024090193212032318, -0.021037623286247253, 0.005877187941223383, -0.013042798265814781, -0.0022217025980353355, -0.017522543668746948, 0.012481178157031536, -0.0006355390651151538, 0.004017234779894352, 0.24145695567131042, 0.006864978466182947, -0.0040866113267838955, 0.011364545673131943, -0.023984476923942566, 0.007188735995441675, 0.03734442591667175, 0.00950789637863636, -0.028702083975076675, 0.015672501176595688, -0.01670323871076107, -0.021605851128697395, 0.01712610572576523, 0.0035844568628817797, 0.007340703625231981, -0.011556156910955906, -0.029653534293174744, -0.0013875317526981235, -0.002456261543557048, -0.025808090344071388, 0.01828899048268795, -0.011853485368192196, -0.029098521918058395, -0.015461067669093609, 0.01877792924642563, 0.036260828375816345, 0.001434608711861074, 0.0008007214055396616, 0.010604706592857838, 0.005728523712605238, -0.0016047465614974499, -0.009871297515928745, 0.0009522762265987694, -0.011945987120270729, -0.027644917368888855, 0.0025983182713389397, 0.018738284707069397, -0.003311906009912491, -0.003343290649354458, -0.007049982436001301, 0.01822291687130928, 0.0036868699826300144, 0.011483476497232914, -0.007915537804365158, -0.021235842257738113, 0.05407409369945526, -0.01617465540766716, 0.0019029006361961365, -0.004714304115623236, 0.02056189812719822, -0.004892701283097267, -0.002264650072902441, 0.01961044780910015, 0.01904222182929516, -0.024962356314063072, 0.0018219612538814545, 0.011615622788667679, 0.007829642854630947, -0.007466242182999849, 0.01398103404790163, 0.00755213713273406, 0.02009938843548298, -0.02432805672287941, 0.01046595349907875, 0.00507440185174346, 0.012481178157031536, -0.011311687529087067, -0.004631713032722473, -0.0025933629367500544, -0.04506174474954605, -0.0015923578757792711, -0.008549838326871395, -0.021737996488809586, 0.00784285832196474, -0.026812398806214333, -0.014853197149932384, 0.004948863293975592, 0.03102785162627697, 0.018526852130889893, 0.026852043345570564, -0.0036241007037460804, -0.006055585108697414, 0.006878193002194166, -0.0007602517725899816, -0.0319528728723526, -0.04545818269252777, 0.01411317940801382, 0.0045359074138104916, -0.006713010836392641, -0.02756563015282154, 0.0008073287317529321, -0.0030294442549347878, -0.006765868980437517, 0.003452311037108302, 0.002398447599261999, -0.012071525678038597, -0.015209990553557873, 0.03385577350854874, -0.00861591100692749, -0.005451017525047064, -0.014562475495040417, -0.009177531115710735, 0.0221476498991251, 0.009375750087201595, -0.010043086484074593, 0.013505308888852596, 0.0018450868083164096, 0.01795862428843975, 0.0042848302982747555, -0.028464222326874733, -0.0039610727690160275, -0.032957181334495544, -0.0020251355599611998, 0.0030459624249488115, 0.008153400383889675, -0.0034853476099669933, -0.0031731529161334038, -0.021381203085184097, 0.006422289647161961, -0.005599681753665209, -0.03340647742152214, -0.001721200067549944, 0.002978237811475992, 0.018923290073871613, -0.003947858233004808, -0.003125250106677413, -0.019993672147393227, -0.002821314614266157, 0.005527001339942217, -0.046594638377428055, 0.01637287437915802, -0.015989651903510094, 0.003059177193790674, 0.00932289194315672, -0.012606716714799404, 0.01842113584280014, -0.010043086484074593, 0.001954107079654932, 0.0023241157177835703, 0.017324324697256088, 0.012375461868941784, -0.022068360820412636, 0.01736396923661232, 0.00045342554221861064, 0.007849465124309063, -0.03541509434580803, 0.012983332388103008, 0.02185692824423313, 0.004020538181066513, -0.029600676149129868, -0.008252509869635105, -0.004611891228705645, -0.012976725585758686, 0.01213099155575037, 0.010360237210988998, -0.04413672164082527, -0.04458601772785187, -0.027644917368888855, -0.011258828453719616, -0.009058600291609764, -0.007466242182999849, 0.012487785890698433, 0.018632568418979645, 0.010413095355033875, -0.0025173788890242577, -0.00808732770383358, -0.1692524403333664, 0.02666703797876835, 0.023429464548826218, -0.020376894623041153, 0.005873884540051222, -0.0025751928333193064, 0.033195044845342636, -0.01036684401333332, -0.01672966778278351, 0.024750923737883568, 0.019319728016853333, 0.006128265056759119, -0.02013903111219406, 0.02284802310168743, -0.007512493059039116, 0.013009761460125446, -0.014588904567062855, 0.03290432319045067, 0.019399015232920647, 0.0063694315031170845, 0.03182072937488556, 0.008536623790860176, 0.012798327952623367, 0.002411662368103862, 0.01339959166944027, 0.01505141519010067, 0.003465525573119521, 0.0018913379171863198, 0.0018748196307569742, -0.005150385666638613, 0.008734842762351036, -8.58432031236589e-05, -0.006950873415917158, -0.022623375058174133, 0.0300764013081789, 0.008820737712085247, -0.00755874440073967, -0.002223354298621416, 0.001600616960786283, 0.015897149220108986, 0.02970639429986477, 0.02089226432144642, 0.007789999712258577, 0.008093935437500477, -0.011424011550843716, 0.026230957359075546, 0.014086750335991383, -0.002994755981490016, 0.01379602961242199, -0.02013903111219406, 0.01437747199088335, -0.023099100217223167, -0.0015428031329065561, 0.00021948604262433946, 0.014879626221954823, -0.005137171130627394, 0.015817862004041672, 0.025279507040977478, -0.007611602544784546, -0.014549260959029198, 0.009435215964913368, -0.02227979525923729, 0.029098521918058395, 0.010307378135621548, 0.008503586985170841, -0.013056012801826, -0.014813552610576153, -0.015804646536707878, -0.03205858916044235, 0.021182984113693237, -0.02023153379559517, -0.008100542239844799, 0.01297011785209179, -0.0009754017810337245, 0.0031235981732606888, 0.004443405196070671, -0.004707696847617626, -0.027777064591646194, 0.0005570774665102363, 0.03594367951154709, -0.017086463049054146, 0.013320304453372955, -0.008021255023777485, 0.004902611952275038, -0.010703816078603268, 0.025279507040977478, -0.013505308888852596, 0.015461067669093609, -0.00882734451442957, -0.013379770331084728, 0.023482322692871094, -0.018936503678560257, -0.029600676149129868, -0.012421712279319763, 0.026164883747696877, 0.009910941123962402, 0.004843146540224552, 0.0033019951079040766, 0.0022117916960269213, -0.022517656907439232, 0.007796606980264187, -0.026759540662169456, 0.01180062722414732, -0.020218320190906525, 0.032296452671289444, 0.003075695363804698, 0.011298472993075848, 0.00964004173874855, 0.033353619277477264, -0.010300771333277225, -0.001560147269628942, -0.004195631481707096, 0.004839842673391104, 0.007268023211508989, -0.016544664278626442, 0.0378994382917881, -0.006105139385908842, -0.007083019241690636, 0.0019227225566282868, -0.005424588453024626, 0.058778487145900726, 0.014443544670939445, -0.027222050353884697, 0.001443693763576448, 0.0004645753651857376, 0.0009390616323798895, -0.12326567620038986, -0.004929041489958763, 0.0027453305665403605, -0.0024446987081319094, 0.01943865790963173, 0.00436411751434207, -0.0273277685046196, -0.014496402814984322, -0.009283248335123062, 0.009692899882793427, -0.009362535551190376, -0.013511915691196918, 0.0024678243789821863, -0.019782237708568573, -0.0015386735321953893, -0.010545240715146065, 0.01891007460653782, -0.02412983775138855, -0.01692788675427437, 0.040330920368433, -0.006898014806210995, -0.005543519742786884, -0.010902035050094128, -0.012633145786821842, 0.004862968344241381, -0.01894971914589405, -0.03456936031579971, 0.0010034827282652259, 0.006818727124482393, 0.005305657163262367, -0.0012760336976498365, -0.025557013228535652, 0.0017740584444254637, -0.013102264143526554, 0.0159764364361763, -0.00862912554293871, -0.01150329876691103, 0.011919558048248291, 0.012395283207297325, -0.01868542656302452, -0.0021407632157206535, 0.01168830320239067, 0.01954437606036663, -0.05092902109026909, -0.003052569692954421, -0.00014876735804136842, -0.020614756271243095, 0.017773620784282684, 0.008992526680231094, -0.014760694466531277, -0.017733976244926453, 0.016280371695756912, -0.025160575285553932, -0.013928175903856754, -0.007717319298535585, 0.0004459923366084695, 0.01024791318923235, 0.009527717716991901, 0.008444122038781643, 0.021909786388278008, -0.01577821746468544, -0.0107963178306818, -0.005705398507416248, -0.019055435433983803, 0.015394994989037514, -0.007968396879732609, -0.004935648757964373, -0.0021143341436982155, -0.014060321263968945, -0.05248834192752838, -0.030393552035093307, -0.004734125919640064, -0.028702083975076675, 0.0042484900914132595, 0.0007197820814326406, -0.000820130342617631, -0.013967819511890411, 0.0022745609749108553, 0.011714732274413109, -0.02597988024353981, -0.010175232775509357, -0.004843146540224552, 0.009732544422149658, -0.028147071599960327, 0.015791432932019234, 0.0016509975539520383, 0.0004214214568492025, 0.009646649472415447, -0.0008011343888938427, 0.00010509727144381031, -0.03205858916044235, 0.028332076966762543, -0.00036567242932505906, -0.013743171468377113, -0.007102841045707464, 0.007426598574966192, -0.001273555913940072, -0.0031103836372494698, 0.009091636165976524, -0.0016782527090981603, -0.0084837656468153, 0.0016485198866575956, -0.03895660489797592, 0.021909786388278008, -0.009712722152471542, -0.01256046537309885, 0.0153157077729702, -0.004810110200196505, 0.004344295710325241, -0.025504155084490776, -0.03523009270429611, 0.006055585108697414, -0.009877904318273067, 0.02743348479270935, 0.021235842257738113, -0.016848599538207054, -0.0143246129155159, -0.021473705768585205, 0.005804507527500391, 0.025755232200026512, 0.007003731559962034, 0.02238551154732704, 0.0013412806438282132, -0.009712722152471542, 0.026323458179831505, 0.009930762462317944, -0.010320592671632767, 0.0041758096776902676, -0.022398727014660835, 0.015751788392663002, -0.014020677655935287, -0.0041758096776902676, 0.0028411364182829857, -0.02397126331925392, 0.009085029363632202, 0.020852619782090187, 0.019399015232920647, 0.008008040487766266, 0.004093218594789505, 0.026112025603652, 0.016597522422671318, 0.06099853664636612, -0.009065207093954086, -0.01683538407087326, -0.005424588453024626, -0.03721227869391441, 0.023033026605844498, -0.010915249586105347, -0.02954781800508499, 0.019055435433983803, 0.025557013228535652, 0.004823324736207724, 0.02142084762454033, 0.011397582478821278, -0.006898014806210995, -0.007122662849724293, -0.0038751778192818165, -0.037238709628582, 0.0024446987081319094, 0.008120364509522915, -0.01150329876691103, -0.015024986118078232, 0.03179429844021797, -0.010948286391794682, 0.001663386239670217, -0.02373339980840683, 0.015461067669093609, -0.016399303451180458, 0.01044613216072321, 0.012818150222301483, -0.006871585734188557, -0.033750057220458984, -0.010571670718491077, -0.002182058757171035, 0.011542942374944687, 0.03208502009510994, 0.01110686082392931, 0.0022084880620241165, -0.001183531479910016, 0.03803158551454544, -0.005236280616372824, 0.02944210171699524, -0.009137887507677078, 0.01119275577366352, -0.0027189014945179224, 0.021777641028165817, 0.023521967232227325, 0.0235748253762722, 0.010320592671632767, -0.0016832081601023674, -0.002233265433460474, 0.003980894573032856, -0.003282173303887248, -0.01047916803508997, 0.004853057209402323, -0.01173455361276865, -0.002753589767962694, 0.02716919220983982, -0.007499278523027897, 0.008245903067290783, 0.01924044080078602, 0.01505141519010067, 0.015950007364153862, -0.01472105085849762, 0.009078421629965305, -0.015091059729456902, -0.022174078971147537, 0.04191667214035988, -0.0032408777624368668, -0.028305647894740105, 0.003954465501010418, 0.021103696897625923, 0.005384944379329681, -0.0009572317358106375, -0.010637743398547173, 0.0107963178306818, -0.015077845193445683, 0.018923290073871613, -0.0033862381242215633, -0.013307089917361736, -0.010789711028337479, 0.034410785883665085, -0.00445661973208189, 0.004189024213701487, 0.009547539986670017, -0.022028718143701553, -0.012732255272567272, 0.0244602020829916, 0.012210278771817684, 0.0038091049063950777, 0.006210856139659882, -0.018989363685250282, -0.009098243899643421, -0.01604251004755497, -0.010056301020085812, -0.008126971311867237, 0.011774198152124882, -0.011457047425210476, -0.0014808597043156624, 0.019887953996658325, -0.036234401166439056, 0.02089226432144642, -0.002771759871393442, -0.006891407538205385, -0.011014359071850777, 0.013089049607515335, 0.01961044780910015, 0.014126394875347614, -0.007968396879732609, -0.012203671969473362, -0.0037496392615139484, 0.015011771582067013, -0.0013957908377051353, 0.029785681515932083, -0.028939947485923767, -0.033591482788324356, -0.012012060731649399, 0.009018956683576107, -0.006191034335643053, 0.015157132409512997, 0.023482322692871094, 0.018315419554710388, 0.008642340078949928, 0.02327089011669159, 0.011001144535839558, -0.021011194214224815, 0.0073539181612432, 0.037978727370500565, -0.017271466553211212, -0.006696492433547974, -0.038507308810949326, -0.004694482311606407, 0.005943261086940765, -0.0559769943356514, -0.024869853630661964, 0.01431139837950468, -0.03377648815512657, -0.009805223904550076, -0.027089904993772507, -0.0004559032677207142, 0.013128693215548992, -0.012534036301076412, 0.003640618873760104, -0.02175121195614338, 0.0084837656468153, 0.004426886793226004, -0.003792586736381054, -0.004998418036848307, -0.0017971839988604188, -0.02838493511080742]}}, rets=[{'node': {'id_': '6769caf1-50b8-4ec0-bfb8-2d7efb033603', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '49ef0a5d-3353-4ef2-9929-ffafbe0170b8', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '4862703e0f236d8e03e59b71b7dbcbc09bf44ec07c7cb1c3cd83ef90b8b04eb3'}, : {'node_id': '760bcb7b-57a7-4506-a196-32d9b33f9bc1', 'node_type': , 'metadata': {}, 'hash': '0b4f822ad12b07d089691e8fe465cfd1a8eb40b43de3358e4fe2ed3606b289e6'}}, 'text': 'I remember taking the boys to the coast on a sunny day in 2015 and figuring out how to deal with some problem involving continuations while I watched them play in the tide pools. It felt like I was doing life right. I remember that because I was slightly dismayed at how novel it felt. The good news is that I had more moments like this over the next few years.\\n\\nIn the summer of 2016 we moved to England.', 'start_char_idx': 65997, 'end_char_idx': 66402, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8211530782069452}, {'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}, {'node': {'id_': 'cc2564e3-e84e-40da-bad8-8824bfac2b58', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': 'f766341d-f7a2-457d-8a43-479d05d4165e', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '41fe2d64055f5b043c933f59e6790309163a302c1596cb142ed78be8f0658ffb'}, : {'node_id': '3712dfd4-6b02-4dc8-a87b-1f3211386ca0', 'node_type': , 'metadata': {}, 'hash': '3779908c352fae99eed78144efa64e8a8a6e91708317acae776ad913bbcc456c'}}, 'text': \"Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them.\\n\\nWhen I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been.\", 'start_char_idx': 38502, 'end_char_idx': 38813, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8174469441429402}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 85388), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 310901)), pid=86042, tid=4683125), RecordAppCall(call_id='ddc8a130-aca4-430e-bd55-489f13aa547a', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='retrieve')), RecordAppCallMethod(path=Lens().app.query_engine._retriever, method=Method(obj=Obj(cls=llama_index.core.indices.vector_store.retrievers.retriever.VectorIndexRetriever, id=14240242384, init_bindings=None), name='retrieve'))], args={'str_or_query_bundle': 'What did the author do growing up?'}, rets=[{'node': {'id_': '6769caf1-50b8-4ec0-bfb8-2d7efb033603', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '49ef0a5d-3353-4ef2-9929-ffafbe0170b8', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '4862703e0f236d8e03e59b71b7dbcbc09bf44ec07c7cb1c3cd83ef90b8b04eb3'}, : {'node_id': '760bcb7b-57a7-4506-a196-32d9b33f9bc1', 'node_type': , 'metadata': {}, 'hash': '0b4f822ad12b07d089691e8fe465cfd1a8eb40b43de3358e4fe2ed3606b289e6'}}, 'text': 'I remember taking the boys to the coast on a sunny day in 2015 and figuring out how to deal with some problem involving continuations while I watched them play in the tide pools. It felt like I was doing life right. I remember that because I was slightly dismayed at how novel it felt. The good news is that I had more moments like this over the next few years.\\n\\nIn the summer of 2016 we moved to England.', 'start_char_idx': 65997, 'end_char_idx': 66402, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8211530782069452}, {'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}, {'node': {'id_': 'cc2564e3-e84e-40da-bad8-8824bfac2b58', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': 'f766341d-f7a2-457d-8a43-479d05d4165e', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '41fe2d64055f5b043c933f59e6790309163a302c1596cb142ed78be8f0658ffb'}, : {'node_id': '3712dfd4-6b02-4dc8-a87b-1f3211386ca0', 'node_type': , 'metadata': {}, 'hash': '3779908c352fae99eed78144efa64e8a8a6e91708317acae776ad913bbcc456c'}}, 'text': \"Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them.\\n\\nWhen I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been.\", 'start_char_idx': 38502, 'end_char_idx': 38813, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8174469441429402}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 30568), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 311693)), pid=86042, tid=4683125), RecordAppCall(call_id='fdd10e06-0ffe-437d-b301-521aa8695190', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='retrieve'))], args={'query_bundle': 'What did the author do growing up?'}, rets=[{'node': {'id_': '6769caf1-50b8-4ec0-bfb8-2d7efb033603', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '49ef0a5d-3353-4ef2-9929-ffafbe0170b8', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '4862703e0f236d8e03e59b71b7dbcbc09bf44ec07c7cb1c3cd83ef90b8b04eb3'}, : {'node_id': '760bcb7b-57a7-4506-a196-32d9b33f9bc1', 'node_type': , 'metadata': {}, 'hash': '0b4f822ad12b07d089691e8fe465cfd1a8eb40b43de3358e4fe2ed3606b289e6'}}, 'text': 'I remember taking the boys to the coast on a sunny day in 2015 and figuring out how to deal with some problem involving continuations while I watched them play in the tide pools. It felt like I was doing life right. I remember that because I was slightly dismayed at how novel it felt. The good news is that I had more moments like this over the next few years.\\n\\nIn the summer of 2016 we moved to England.', 'start_char_idx': 65997, 'end_char_idx': 66402, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8211530782069452}, {'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}, {'node': {'id_': 'cc2564e3-e84e-40da-bad8-8824bfac2b58', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': 'f766341d-f7a2-457d-8a43-479d05d4165e', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '41fe2d64055f5b043c933f59e6790309163a302c1596cb142ed78be8f0658ffb'}, : {'node_id': '3712dfd4-6b02-4dc8-a87b-1f3211386ca0', 'node_type': , 'metadata': {}, 'hash': '3779908c352fae99eed78144efa64e8a8a6e91708317acae776ad913bbcc456c'}}, 'text': \"Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them.\\n\\nWhen I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been.\", 'start_char_idx': 38502, 'end_char_idx': 38813, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8174469441429402}], error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 49, 975450), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 312391)), pid=86042, tid=4683125), RecordAppCall(call_id='f15702ed-67f7-4a53-88e0-319ff9f20096', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=14515813456, init_bindings=None), name='__call__'))], args={'args': ['What did the author do growing up?', \"Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them.\\n\\nWhen I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been.\"]}, rets=0.2, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 368430), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 643809)), pid=86042, tid=4691946), RecordAppCall(call_id='52a41ee2-810e-4389-8e8e-7077b88e1ce6', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=14515813456, init_bindings=None), name='__call__'))], args={'args': ['What did the author do growing up?', 'I remember taking the boys to the coast on a sunny day in 2015 and figuring out how to deal with some problem involving continuations while I watched them play in the tide pools. It felt like I was doing life right. I remember that because I was slightly dismayed at how novel it felt. The good news is that I had more moments like this over the next few years.\\n\\nIn the summer of 2016 we moved to England.']}, rets=0.4, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 333154), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 673338)), pid=86042, tid=4691944), RecordAppCall(call_id='ffd31ad5-405b-4018-beec-810a6409e12a', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.feedback, method=Method(obj=Obj(cls=trulens_eval.feedback.feedback.Feedback, id=14515813456, init_bindings=None), name='__call__'))], args={'args': ['What did the author do growing up?', \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\"]}, rets=0.8, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 355434), end_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 761848)), pid=86042, tid=4691945), RecordAppCall(call_id='0f85b8d3-f9f1-4361-9daf-ce0d300fae01', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='synthesize')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer, method=Method(obj=Obj(cls=llama_index.core.response_synthesizers.compact_and_refine.CompactAndRefine, id=14237596304, init_bindings=None), name='get_response')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer, method=Method(obj=Obj(cls=llama_index.core.response_synthesizers.refine.Refine, id=14237596304, init_bindings=None), name='get_response')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer._llm, method=Method(obj=Obj(cls=llama_index.llms.openai.base.OpenAI, id=14162967920, init_bindings=None), name='chat'))], args={'_self': {'callback_manager': {'__tru_non_serialized_object': {'cls': {'name': 'CallbackManager', 'module': {'package_name': 'llama_index.core.callbacks', 'module_name': 'llama_index.core.callbacks.base'}, 'bases': None}, 'id': 14159290320, 'init_bindings': None}}, 'system_prompt': None, 'messages_to_prompt': {'__tru_non_serialized_object': {'cls': {'name': 'function', 'module': {'package_name': '', 'module_name': 'builtins'}, 'bases': None}, 'id': 14147734400, 'init_bindings': None}}, 'completion_to_prompt': {'__tru_non_serialized_object': {'cls': {'name': 'function', 'module': {'package_name': '', 'module_name': 'builtins'}, 'bases': None}, 'id': 14148483680, 'init_bindings': None}}, 'output_parser': None, 'pydantic_program_mode': , 'query_wrapper_prompt': None, 'model': 'gpt-3.5-turbo', 'temperature': 0.1, 'max_tokens': None, 'logprobs': None, 'top_logprobs': 0, 'additional_kwargs': {}, 'max_retries': 3, 'timeout': 60.0, 'default_headers': None, 'reuse_client': True, 'api_key': 'sk-...', 'api_base': 'https://api.openai.com/v1', 'api_version': ''}, 'messages': [{'role': , 'content': \"You are an expert Q&A system that is trusted around the world.\\nAlways answer the query using the provided context information, and not prior knowledge.\\nSome rules to follow:\\n1. Never directly reference the given context in your answer.\\n2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines.\", 'additional_kwargs': {}}, {'role': , 'content': \"Context information is below.\\n---------------------\\nfile_path: /Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt\\n\\nWhat I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\\n---------------------\\nGiven the context information and not prior knowledge, answer the query.\\nQuery: What did the author do growing up?\\nAnswer: \", 'additional_kwargs': {}}]}, rets={'message': {'role': , 'content': 'The author worked on writing and programming outside of school before college.', 'additional_kwargs': {}}, 'raw': {'id': 'chatcmpl-9Yj7fiT7YtVYd55GklD6wKGVyfqOg', 'choices': [{'finish_reason': 'stop', 'index': 0, 'logprobs': None, 'message': {'content': 'The author worked on writing and programming outside of school before college.', 'role': 'assistant', 'function_call': None, 'tool_calls': None}}], 'created': 1718063631, 'model': 'gpt-3.5-turbo-0125', 'object': 'chat.completion', 'system_fingerprint': None, 'usage': {'completion_tokens': 13, 'prompt_tokens': 229, 'total_tokens': 242}}, 'delta': None, 'logprobs': None, 'additional_kwargs': {}}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 23766), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 606421)), pid=86042, tid=4683125), RecordAppCall(call_id='12e8067e-5181-432b-bc2e-6f69061b5a7e', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='synthesize')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer, method=Method(obj=Obj(cls=llama_index.core.response_synthesizers.compact_and_refine.CompactAndRefine, id=14237596304, init_bindings=None), name='get_response')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer, method=Method(obj=Obj(cls=llama_index.core.response_synthesizers.refine.Refine, id=14237596304, init_bindings=None), name='get_response'))], args={'query_str': 'What did the author do growing up?', 'text_chunks': [\"file_path: /Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt\\n\\nWhat I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\"], 'prev_response': None}, rets='The author worked on writing and programming outside of school before college.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 946216), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 606708)), pid=86042, tid=4683125), RecordAppCall(call_id='96158ac9-b5f3-454e-bcb2-1fdbfbc663d4', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='synthesize')), RecordAppCallMethod(path=Lens().app.query_engine._response_synthesizer, method=Method(obj=Obj(cls=llama_index.core.response_synthesizers.compact_and_refine.CompactAndRefine, id=14237596304, init_bindings=None), name='get_response'))], args={'query_str': 'What did the author do growing up?', 'text_chunks': [\"file_path: /Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt\\n\\nWhat I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\"]}, rets='The author worked on writing and programming outside of school before college.', error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 873663), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 606734)), pid=86042, tid=4683125), RecordAppCall(call_id='01ebd147-9fc6-417d-b761-857b6412b864', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query')), RecordAppCallMethod(path=Lens().app.query_engine, method=Method(obj=Obj(cls=llama_index.core.query_engine.retriever_query_engine.RetrieverQueryEngine, id=14369330640, init_bindings=None), name='synthesize'))], args={'query_bundle': 'What did the author do growing up?', 'nodes': [{'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}]}, rets={'response': 'The author worked on writing and programming outside of school before college.', 'source_nodes': [{'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}], 'metadata': {'3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}}}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 50, 804923), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 607057)), pid=86042, tid=4683125), RecordAppCall(call_id='43b44624-e2ea-45e1-bde2-dee1b04f76fb', stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=trulens_eval.guardrails.llama.WithFeedbackFilterNodes, id=14369330640, init_bindings=None), name='query'))], args={'query': 'What did the author do growing up?'}, rets={'response': 'The author worked on writing and programming outside of school before college.', 'source_nodes': [{'node': {'id_': '3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e', 'embedding': None, 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'excluded_embed_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'excluded_llm_metadata_keys': ['file_name', 'file_type', 'file_size', 'creation_date', 'last_modified_date', 'last_accessed_date'], 'relationships': {: {'node_id': '7ca0bef3-9d52-437d-86b9-9e48eae8e467', 'node_type': , 'metadata': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}, 'hash': '55c51f960a8f2448ccead00ea194bb3971e448ce14f25d9928a79705d8f7ef23'}, : {'node_id': '3976e410-ba26-4cc0-bc98-0475d9f33c1b', 'node_type': , 'metadata': {}, 'hash': 'e99647d69b84b0a13c59268b58406bb00652b41196a27d7b46b56e5d713018ed'}}, 'text': \"What I Worked On\\n\\nFebruary 2021\\n\\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\", 'start_char_idx': 2, 'end_char_idx': 373, 'text_template': '{metadata_str}\\n\\n{content}', 'metadata_template': '{key}: {value}', 'metadata_seperator': '\\n'}, 'score': 0.8207477152744018}], 'metadata': {'3e1e67fb-89f3-4af8-83b7-f9a12cb7ec1e': {'file_path': '/Users/jreini/Desktop/development/trulens/trulens_eval/examples/quickstart/data/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-10', 'last_modified_date': '2024-06-10'}}}, error=None, perf=Perf(start_time=datetime.datetime(2024, 6, 10, 19, 53, 49, 921442), end_time=datetime.datetime(2024, 6, 10, 19, 53, 51, 607337)), pid=86042, tid=4683125)], feedback_and_future_results=[(FeedbackDefinition(relevance,\n", + "\tselectors={'prompt': Lens().__record__.main_input, 'response': Lens().__record__.main_output},\n", + "\tif_exists=None\n", + "), ), (FeedbackDefinition(context_relevance_with_cot_reasons,\n", + "\tselectors={'question': Lens().__record__.main_input, 'context': Lens().__record__.app.query.rets.source_nodes[:].node.text},\n", + "\tif_exists=None\n", + "), ), (FeedbackDefinition(groundedness_measure_with_cot_reasons,\n", + "\tselectors={'source': Lens().__record__.app.query.rets.source_nodes[:].node.text.collect(), 'statement': Lens().__record__.main_output},\n", + "\tif_exists=None\n", + "), )], feedback_results=[, , ])" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# The record of the app invocation can be retrieved from the `recording`:\n", "\n", @@ -233,18 +513,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting dashboard ...\n", + "Config file already exists. Skipping writing process.\n", + "Credentials file already exists. Skipping writing process.\n", + "Dashboard already running at path: Network URL: http://192.168.4.206:8501\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tru.run_dashboard()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "relevance 0.8\n", + "context_relevance_with_cot_reasons 0.8\n", + "groundedness_measure_with_cot_reasons 1.0\n" + ] + } + ], "source": [ "# The results of the feedback functions can be rertireved from\n", "# `Record.feedback_results` or using the `wait_for_feedback_result` method. The\n", @@ -261,22 +573,338 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
app_idapp_jsontyperecord_idinputoutputtagsrecord_jsoncost_jsonperf_jsontsrelevancegroundedness_measure_with_cot_reasonscontext_relevance_with_cot_reasonsrelevance_callsgroundedness_measure_with_cot_reasons_callscontext_relevance_with_cot_reasons_callslatencytotal_tokenstotal_cost
0LlamaIndex_App1{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...RetrieverQueryEngine(llama_index.core.query_en...record_hash_38c564bcce22598a80a36d4f5ebc7e5f\"What did the author do growing up?\"\"The author worked on writing and programming ...-{\"record_id\": \"record_hash_38c564bcce22598a80a...{\"n_requests\": 2, \"n_successful_requests\": 2, ...{\"start_time\": \"2024-06-10T19:46:40.006586\", \"...2024-06-10T19:46:40.9517700.71.00.4[{'args': {'prompt': 'What did the author do g...[{'args': {'source': ['I remember taking the b...[{'args': {'question': 'What did the author do...04790.000713
1LlamaIndex_App1{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...RetrieverQueryEngine(llama_index.core.query_en...record_hash_43a605c17473821547ccb9d02575f6bf\"What did the author do growing up?\"\"The author worked on writing and programming ...-{\"record_id\": \"record_hash_43a605c17473821547c...{\"n_requests\": 2, \"n_successful_requests\": 2, ...{\"start_time\": \"2024-06-10T19:53:42.091728\", \"...2024-06-10T19:53:43.1525120.71.00.4[{'args': {'prompt': 'What did the author do g...[{'args': {'source': ['I remember taking the b...[{'args': {'question': 'What did the author do...14790.000713
2LlamaIndex_App1_Filtered{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...WithFeedbackFilterNodes(trulens_eval.guardrail...record_hash_e5b3eb91b4c11ee475ade96acb15f906\"What did the author do growing up?\"\"The author focused on writing and programming...-{\"record_id\": \"record_hash_e5b3eb91b4c11ee475a...{\"n_requests\": 5, \"n_successful_requests\": 5, ...{\"start_time\": \"2024-06-10T19:46:46.683277\", \"...2024-06-10T19:46:48.4442840.71.00.8[{'args': {'prompt': 'What did the author do g...[{'args': {'source': [\"What I Worked On\\n\\nFeb...[{'args': {'question': 'What did the author do...013220.001979
3LlamaIndex_App1_Filtered{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...WithFeedbackFilterNodes(trulens_eval.guardrail...record_hash_404429446b4fc7f465daf0980c6fedc9\"What did the author do growing up?\"\"The author worked on writing and programming ...-{\"record_id\": \"record_hash_404429446b4fc7f465d...{\"n_requests\": 5, \"n_successful_requests\": 5, ...{\"start_time\": \"2024-06-10T19:53:49.921442\", \"...2024-06-10T19:53:51.6076440.81.00.8[{'args': {'prompt': 'What did the author do g...[{'args': {'source': [\"What I Worked On\\n\\nFeb...[{'args': {'question': 'What did the author do...113220.001979
\n", + "
" + ], + "text/plain": [ + " app_id \\\n", + "0 LlamaIndex_App1 \n", + "1 LlamaIndex_App1 \n", + "2 LlamaIndex_App1_Filtered \n", + "3 LlamaIndex_App1_Filtered \n", + "\n", + " app_json \\\n", + "0 {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul... \n", + "1 {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul... \n", + "2 {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul... \n", + "3 {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul... \n", + "\n", + " type \\\n", + "0 RetrieverQueryEngine(llama_index.core.query_en... \n", + "1 RetrieverQueryEngine(llama_index.core.query_en... \n", + "2 WithFeedbackFilterNodes(trulens_eval.guardrail... \n", + "3 WithFeedbackFilterNodes(trulens_eval.guardrail... \n", + "\n", + " record_id \\\n", + "0 record_hash_38c564bcce22598a80a36d4f5ebc7e5f \n", + "1 record_hash_43a605c17473821547ccb9d02575f6bf \n", + "2 record_hash_e5b3eb91b4c11ee475ade96acb15f906 \n", + "3 record_hash_404429446b4fc7f465daf0980c6fedc9 \n", + "\n", + " input \\\n", + "0 \"What did the author do growing up?\" \n", + "1 \"What did the author do growing up?\" \n", + "2 \"What did the author do growing up?\" \n", + "3 \"What did the author do growing up?\" \n", + "\n", + " output tags \\\n", + "0 \"The author worked on writing and programming ... - \n", + "1 \"The author worked on writing and programming ... - \n", + "2 \"The author focused on writing and programming... - \n", + "3 \"The author worked on writing and programming ... - \n", + "\n", + " record_json \\\n", + "0 {\"record_id\": \"record_hash_38c564bcce22598a80a... \n", + "1 {\"record_id\": \"record_hash_43a605c17473821547c... \n", + "2 {\"record_id\": \"record_hash_e5b3eb91b4c11ee475a... \n", + "3 {\"record_id\": \"record_hash_404429446b4fc7f465d... \n", + "\n", + " cost_json \\\n", + "0 {\"n_requests\": 2, \"n_successful_requests\": 2, ... \n", + "1 {\"n_requests\": 2, \"n_successful_requests\": 2, ... \n", + "2 {\"n_requests\": 5, \"n_successful_requests\": 5, ... \n", + "3 {\"n_requests\": 5, \"n_successful_requests\": 5, ... \n", + "\n", + " perf_json \\\n", + "0 {\"start_time\": \"2024-06-10T19:46:40.006586\", \"... \n", + "1 {\"start_time\": \"2024-06-10T19:53:42.091728\", \"... \n", + "2 {\"start_time\": \"2024-06-10T19:46:46.683277\", \"... \n", + "3 {\"start_time\": \"2024-06-10T19:53:49.921442\", \"... \n", + "\n", + " ts relevance \\\n", + "0 2024-06-10T19:46:40.951770 0.7 \n", + "1 2024-06-10T19:53:43.152512 0.7 \n", + "2 2024-06-10T19:46:48.444284 0.7 \n", + "3 2024-06-10T19:53:51.607644 0.8 \n", + "\n", + " groundedness_measure_with_cot_reasons context_relevance_with_cot_reasons \\\n", + "0 1.0 0.4 \n", + "1 1.0 0.4 \n", + "2 1.0 0.8 \n", + "3 1.0 0.8 \n", + "\n", + " relevance_calls \\\n", + "0 [{'args': {'prompt': 'What did the author do g... \n", + "1 [{'args': {'prompt': 'What did the author do g... \n", + "2 [{'args': {'prompt': 'What did the author do g... \n", + "3 [{'args': {'prompt': 'What did the author do g... \n", + "\n", + " groundedness_measure_with_cot_reasons_calls \\\n", + "0 [{'args': {'source': ['I remember taking the b... \n", + "1 [{'args': {'source': ['I remember taking the b... \n", + "2 [{'args': {'source': [\"What I Worked On\\n\\nFeb... \n", + "3 [{'args': {'source': [\"What I Worked On\\n\\nFeb... \n", + "\n", + " context_relevance_with_cot_reasons_calls latency total_tokens \\\n", + "0 [{'args': {'question': 'What did the author do... 0 479 \n", + "1 [{'args': {'question': 'What did the author do... 1 479 \n", + "2 [{'args': {'question': 'What did the author do... 0 1322 \n", + "3 [{'args': {'question': 'What did the author do... 1 1322 \n", + "\n", + " total_cost \n", + "0 0.000713 \n", + "1 0.000713 \n", + "2 0.001979 \n", + "3 0.001979 " + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "records, feedback = tru.get_records_and_feedback(app_ids=[\"LlamaIndex_App1\"])\n", + "records, feedback = tru.get_records_and_feedback(app_ids=[])\n", "\n", "records.head()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
groundedness_measure_with_cot_reasonscontext_relevance_with_cot_reasonsrelevancelatencytotal_cost
app_id
LlamaIndex_App1_Filtered1.00.80.750.50.001979
LlamaIndex_App11.00.40.700.50.000713
\n", + "
" + ], + "text/plain": [ + " groundedness_measure_with_cot_reasons \\\n", + "app_id \n", + "LlamaIndex_App1_Filtered 1.0 \n", + "LlamaIndex_App1 1.0 \n", + "\n", + " context_relevance_with_cot_reasons relevance \\\n", + "app_id \n", + "LlamaIndex_App1_Filtered 0.8 0.75 \n", + "LlamaIndex_App1 0.4 0.70 \n", + "\n", + " latency total_cost \n", + "app_id \n", + "LlamaIndex_App1_Filtered 0.5 0.001979 \n", + "LlamaIndex_App1 0.5 0.000713 " + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "tru.get_leaderboard(app_ids=[\"LlamaIndex_App1\"])" + "tru.get_leaderboard(app_ids=[])" ] }, { @@ -289,9 +917,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting dashboard ...\n", + "Config file already exists. Skipping writing process.\n", + "Credentials file already exists. Skipping writing process.\n", + "Dashboard already running at path: Network URL: http://192.168.4.206:8501\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tru.run_dashboard() # open a local streamlit app to explore\n", "\n", @@ -323,7 +973,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.11.9" }, "vscode": { "interpreter": { diff --git a/trulens_eval/examples/quickstart/quickstart.ipynb b/trulens_eval/examples/quickstart/quickstart.ipynb index 6f3a3fca0..59e594f3d 100644 --- a/trulens_eval/examples/quickstart/quickstart.ipynb +++ b/trulens_eval/examples/quickstart/quickstart.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -24,13 +24,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", - "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n", - "os.environ[\"HUGGINGFACE_API_KEY\"] = \"hf_...\"" + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" ] }, { @@ -44,16 +43,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "university_info = \"\"\"\n", + "uw_info = \"\"\"\n", "The University of Washington, founded in 1861 in Seattle, is a public research university\n", "with over 45,000 students across three campuses in Seattle, Tacoma, and Bothell.\n", "As the flagship institution of the six public universities in Washington state,\n", "UW encompasses over 500 buildings and 20 million square feet of space,\n", "including one of the largest library systems in the world.\n", + "\"\"\"\n", + "\n", + "wsu_info = \"\"\"\n", + "Washington State University, commonly known as WSU, founded in 1890, is a public research university in Pullman, Washington.\n", + "With multiple campuses across the state, it is the state's second largest institution of higher education.\n", + "WSU is known for its programs in veterinary medicine, agriculture, engineering, architecture, and pharmacy.\n", + "\"\"\"\n", + "\n", + "seattle_info = \"\"\"\n", + "Seattle, a city on Puget Sound in the Pacific Northwest, is surrounded by water, mountains and evergreen forests, and contains thousands of acres of parkland.\n", + "It's home to a large tech industry, with Microsoft and Amazon headquartered in its metropolitan area.\n", + "The futuristic Space Needle, a legacy of the 1962 World's Fair, is its most iconic landmark.\n", + "\"\"\"\n", + "\n", + "starbucks_info = \"\"\"\n", + "Starbucks Corporation is an American multinational chain of coffeehouses and roastery reserves headquartered in Seattle, Washington.\n", + "As the world's largest coffeehouse chain, Starbucks is seen to be the main representation of the United States' second wave of coffee culture.\n", "\"\"\"" ] }, @@ -68,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -80,7 +96,7 @@ "\n", "\n", "chroma_client = chromadb.Client()\n", - "vector_store = chroma_client.get_or_create_collection(name=\"Universities\",\n", + "vector_store = chroma_client.get_or_create_collection(name=\"Washington\",\n", " embedding_function=embedding_function)" ] }, @@ -90,18 +106,21 @@ "collapsed": false }, "source": [ - "Add the university_info to the embedding database." + "Populate the vector store." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ - "vector_store.add(\"uni_info\", documents=university_info)" + "vector_store.add(\"uw_info\", documents=uw_info)\n", + "vector_store.add(\"wsu_info\", documents=wsu_info)\n", + "vector_store.add(\"seattle_info\", documents=seattle_info)\n", + "vector_store.add(\"starbucks_info\", documents=starbucks_info)" ] }, { @@ -115,18 +134,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/snowday/lib/python3.11/site-packages/_distutils_hack/__init__.py:26: UserWarning: Setuptools is replacing distutils.\n", + " warnings.warn(\"Setuptools is replacing distutils.\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "πŸ¦‘ Tru initialized with db url sqlite:///default.sqlite .\n", + "πŸ›‘ Secret keys may be written to the database. See the `database_redact_keys` option of `Tru` to prevent this.\n" + ] + } + ], "source": [ "from trulens_eval import Tru\n", "from trulens_eval.tru_custom_app import instrument\n", - "tru = Tru()" + "tru = Tru()\n", + "tru.reset_database()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -140,10 +177,11 @@ " Retrieve relevant text from vector store.\n", " \"\"\"\n", " results = vector_store.query(\n", - " query_texts=query,\n", - " n_results=2\n", - " )\n", - " return results['documents']\n", + " query_texts=query,\n", + " n_results=4\n", + " )\n", + " # Flatten the list of lists into a single list\n", + " return [doc for sublist in results['documents'] for doc in sublist]\n", "\n", " @instrument\n", " def generate_completion(self, query: str, context_str: list) -> str:\n", @@ -187,9 +225,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In Groundedness, input source will be set to __record__.app.retrieve.rets.collect() .\n", + "βœ… In Groundedness, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In Answer Relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In Answer Relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n", + "βœ… In Context Relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In Context Relevance, input context will be set to __record__.app.retrieve.rets[:] .\n" + ] + } + ], "source": [ "from trulens_eval import Feedback, Select\n", "from trulens_eval.feedback.provider.openai import OpenAI\n", @@ -207,15 +258,15 @@ "# Question/answer relevance between overall question and answer.\n", "f_answer_relevance = (\n", " Feedback(provider.relevance_with_cot_reasons, name = \"Answer Relevance\")\n", - " .on(Select.RecordCalls.retrieve.args.query)\n", + " .on_input()\n", " .on_output()\n", ")\n", "\n", "# Context relevance between question and each context chunk.\n", "f_context_relevance = (\n", " Feedback(provider.context_relevance_with_cot_reasons, name = \"Context Relevance\")\n", - " .on(Select.RecordCalls.retrieve.args.query)\n", - " .on(Select.RecordCalls.retrieve.rets)\n", + " .on_input()\n", + " .on(Select.RecordCalls.retrieve.rets[:])\n", " .aggregate(np.mean) # choose a different aggregation method if you wish\n", ")" ] @@ -230,7 +281,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -250,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -258,20 +309,464 @@ " rag.query(\"When was the University of Washington founded?\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check results\n", + "\n", + "We can view results in the leaderboard." + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
RAG v11.01.00.41.00.000511
\n", + "
" + ], + "text/plain": [ + " Groundedness Answer Relevance Context Relevance latency total_cost\n", + "app_id \n", + "RAG v1 1.0 1.0 0.4 1.0 0.000511" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tru.get_leaderboard()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0When was the University of Washington founded?\\nThe University of Washington, founded in 186...1.0
1When was the University of Washington founded?\\nWashington State University, commonly known ...0.2
2When was the University of Washington founded?\\nSeattle, a city on Puget Sound in the Pacifi...0.2
3When was the University of Washington founded?\\nStarbucks Corporation is an American multina...0.2
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 When was the University of Washington founded? \n", + "1 When was the University of Washington founded? \n", + "2 When was the University of Washington founded? \n", + "3 When was the University of Washington founded? \n", + "\n", + " context ret \n", + "0 \\nThe University of Washington, founded in 186... 1.0 \n", + "1 \\nWashington State University, commonly known ... 0.2 \n", + "2 \\nSeattle, a city on Puget Sound in the Pacifi... 0.2 \n", + "3 \\nStarbucks Corporation is an American multina... 0.2 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "last_record = recording.records[-1]\n", + "\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'Context Relevance')" + ] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "tru.get_leaderboard(app_ids=[\"RAG v1\"])" + "## Use guardrails\n", + "\n", + "In addition to making informed iteration, we can also directly use feedback results as guardrails at inference time. In particular, here we show how to use the context relevance score as a guardrail to filter out irrelevant context before it gets passed to the LLM. This both reduces hallucination and improves efficiency.\n", + "\n", + "To do so, we'll rebuild our RAG using the @context-filter decorator on the method we want to filter, and pass in the feedback function and threshold to use for guardrailing." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "βœ… In Context Relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n", + "βœ… In Context Relevance, input context will be set to __record__.app.retrieve.rets .\n" + ] + } + ], + "source": [ + "# note: feedback function used for guardrail must only return a score, not also reasons\n", + "f_context_relevance_score = (\n", + " Feedback(provider.context_relevance, name = \"Context Relevance\")\n", + " .on_input()\n", + " .on(Select.RecordCalls.retrieve.rets)\n", + ")\n", + "\n", + "from trulens_eval.guardrails.base import context_filter\n", + "\n", + "class filtered_RAG_from_scratch:\n", + " @instrument\n", + " @context_filter(f_context_relevance_score, 0.5)\n", + " def retrieve(self, query: str) -> list:\n", + " \"\"\"\n", + " Retrieve relevant text from vector store.\n", + " \"\"\"\n", + " results = vector_store.query(\n", + " query_texts=query,\n", + " n_results=4\n", + " )\n", + " return [doc for sublist in results['documents'] for doc in sublist]\n", + "\n", + " @instrument\n", + " def generate_completion(self, query: str, context_str: list) -> str:\n", + " \"\"\"\n", + " Generate answer from context.\n", + " \"\"\"\n", + " completion = oai_client.chat.completions.create(\n", + " model=\"gpt-3.5-turbo\",\n", + " temperature=0,\n", + " messages=\n", + " [\n", + " {\"role\": \"user\",\n", + " \"content\": \n", + " f\"We have provided context information below. \\n\"\n", + " f\"---------------------\\n\"\n", + " f\"{context_str}\"\n", + " f\"\\n---------------------\\n\"\n", + " f\"Given this information, please answer the question: {query}\"\n", + " }\n", + " ]\n", + " ).choices[0].message.content\n", + " return completion\n", + "\n", + " @instrument\n", + " def query(self, query: str) -> str:\n", + " context_str = self.retrieve(query)\n", + " completion = self.generate_completion(query, context_str)\n", + " return completion\n", + "\n", + "filtered_rag = filtered_RAG_from_scratch()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Record and operate as normal" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": {}, "outputs": [], + "source": [ + "from trulens_eval import TruCustomApp\n", + "filtered_tru_rag = TruCustomApp(filtered_rag,\n", + " app_id = 'RAG v2',\n", + " feedbacks = [f_groundedness, f_answer_relevance, f_context_relevance])\n", + "\n", + "with filtered_tru_rag as recording:\n", + " filtered_rag.query(\"when was the university of washington founded?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
GroundednessAnswer RelevanceContext Relevancelatencytotal_cost
app_id
RAG v21.01.01.01.00.002268
RAG v11.01.00.41.00.000511
\n", + "
" + ], + "text/plain": [ + " Groundedness Answer Relevance Context Relevance latency total_cost\n", + "app_id \n", + "RAG v2 1.0 1.0 1.0 1.0 0.002268\n", + "RAG v1 1.0 1.0 0.4 1.0 0.000511" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tru.get_leaderboard(app_ids=[])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See the power of filtering!" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncontextret
0when was the university of washington founded?\\nThe University of Washington, founded in 186...1.0
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 when was the university of washington founded? \n", + "\n", + " context ret \n", + "0 \\nThe University of Washington, founded in 186... 1.0 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "last_record = recording.records[-1]\n", + "\n", + "from trulens_eval.utils.display import get_feedback_result\n", + "get_feedback_result(last_record, 'Context Relevance')" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting dashboard ...\n", + "Config file already exists. Skipping writing process.\n", + "Credentials file already exists. Skipping writing process.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ca8f119515894a498deb00bfccb6239d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Accordion(children=(VBox(children=(VBox(children=(Label(value='STDOUT'), Output())), VBox(children=(Label(valu…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dashboard started at http://192.168.4.206:11177 .\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tru.run_dashboard()" ] @@ -293,7 +788,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/trulens_eval/tests/unit/static/test_static.py b/trulens_eval/tests/unit/static/test_static.py index c879de65a..a5a601000 100644 --- a/trulens_eval/tests/unit/static/test_static.py +++ b/trulens_eval/tests/unit/static/test_static.py @@ -28,7 +28,10 @@ optional_mods = dict( pinecone=["trulens_eval.Example_TruBot"], ipywidgets=["trulens_eval.appui"], - llama_index=["trulens_eval.tru_llama", "trulens_eval.utils.llama"], + llama_index=[ + "trulens_eval.tru_llama", "trulens_eval.utils.llama", + "trulens_eval.guardrails.llama" + ], boto3=[ "trulens_eval.feedback.provider.bedrock", "trulens_eval.feedback.provider.endpoint.bedrock" diff --git a/trulens_eval/trulens_eval/guardrails/__init__.py b/trulens_eval/trulens_eval/guardrails/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/trulens_eval/trulens_eval/guardrails/base.py b/trulens_eval/trulens_eval/guardrails/base.py new file mode 100644 index 000000000..5a86e3962 --- /dev/null +++ b/trulens_eval/trulens_eval/guardrails/base.py @@ -0,0 +1,60 @@ +from concurrent.futures import as_completed + +from trulens_eval.feedback import Feedback +from trulens_eval.utils.threading import ThreadPoolExecutor + + +class context_filter: + """ + Provides a decorator to filter contexts based on a given feedback and threshold. + + Parameters: + feedback (Feedback): The feedback object to use for filtering. + threshold (float): The minimum feedback value required for a context to be included. + + !!! example + + ```python + feedback = Feedback(provider.groundedness_measure_with_cot_reasons, name="Groundedness") + @context_filter(feedback, 0.5) + def retrieve(query: str) -> list: + results = vector_store.query( + query_texts=query, + n_results=3 + ) + return [doc for sublist in results['documents'] for doc in sublist] + ``` + """ + + def __init__(self, feedback: Feedback, threshold: float): + self.feedback = feedback + self.threshold = threshold + + def __call__(self, func): + + def wrapper(*args, **kwargs): + contexts = func(*args, **kwargs) + with ThreadPoolExecutor(max_workers=max(1, len(contexts))) as ex: + future_to_context = { + ex.submit( + lambda context=context: self.feedback(args[1], context) + ): context for context in contexts + } + filtered = [] + for future in as_completed(future_to_context): + context = future_to_context[future] + result = future.result() + if not isinstance(result, float): + raise ValueError( + "Guardrails can only be used with feedback functions that return a float." + ) + if (self.feedback.higher_is_better and result > self.threshold) or \ + (not self.feedback.higher_is_better and result < self.threshold): + filtered.append(context) + return filtered + + # note: the following information is manually written to the wrapper because @functools.wraps(func) causes breaking of the method. + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + wrapper.__signature__ = func.__signature__ + return wrapper diff --git a/trulens_eval/trulens_eval/guardrails/langchain.py b/trulens_eval/trulens_eval/guardrails/langchain.py new file mode 100644 index 000000000..1defd00bc --- /dev/null +++ b/trulens_eval/trulens_eval/guardrails/langchain.py @@ -0,0 +1,122 @@ +from concurrent.futures import as_completed +from typing import List + +from trulens_eval.feedback import Feedback +from trulens_eval.utils.imports import OptionalImports +from trulens_eval.utils.imports import REQUIREMENT_LANGCHAIN +from trulens_eval.utils.serial import model_dump +from trulens_eval.utils.threading import ThreadPoolExecutor + +with OptionalImports(messages=REQUIREMENT_LANGCHAIN): + import langchain + from langchain.schema import Document + from langchain.vectorstores.base import VectorStoreRetriever + +OptionalImports(messages=REQUIREMENT_LANGCHAIN).assert_installed(langchain) + + +class WithFeedbackFilterDocuments(VectorStoreRetriever): + feedback: Feedback + threshold: float + """ + A VectorStoreRetriever that filters documents using a minimum threshold + on a feedback function before returning them. + + Args: + feedback (Feedback): use this feedback function to score each document. + + threshold (float): and keep documents only if their feedback value is at least this threshold. + + !!! example "Using TruLens guardrail context filters with Langchain" + + ```python + from trulens_eval.guardrails.langchain import WithFeedbackFilterDocuments + + # note: feedback function used for guardrail must only return a score, not also reasons + feedback = Feedback(provider.context_relevance).on_input().on(context) + + filtered_retriever = WithFeedbackFilterDocuments.of_retriever( + retriever=retriever, + feedback=feedback, + threshold=0.5 + ) + + rag_chain = {"context": filtered_retriever | format_docs, "question": RunnablePassthrough()} | prompt | llm | StrOutputParser() + + tru_recorder = TruChain(rag_chain, + app_id='Chain1_ChatApplication_Filtered') + + with tru_recorder as recording: + llm_response = rag_chain.invoke("What is Task Decomposition?") + ``` + """ + + def __init__(self, feedback: Feedback, threshold: float, *args, **kwargs): + super().__init__( + *args, feedback=feedback, threshold=threshold, **kwargs + ) + + # Signature must match + # langchain.schema.retriever.BaseRetriever._get_relevant_documents . + def _get_relevant_documents(self, query: str, *, + run_manager) -> List[Document]: + """ + An internal method to accomplish three tasks: + + 1. Get relevant documents. + 2. Evaluate documents with a specified feedback function. + 3. Filter out documents that do not meet the minimum threshold. + + Args: + query: str - the query string to search for relevant documents. + + run_manager: RunManager - the run manager to handle document retrieval. + + Returns: + - List[Document]: a list of filtered, relevant documents. + """ + # Get relevant docs using super class: + docs = super()._get_relevant_documents(query, run_manager=run_manager) + + # Evaluate the filter on each, in parallel. + with ThreadPoolExecutor(max_workers=max(1, len(docs))) as ex: + future_to_doc = { + ex. + submit(lambda doc=doc: self.feedback(query, doc.page_content)): + doc for doc in docs + } + filtered = [] + for future in as_completed(future_to_doc): + doc = future_to_doc[future] + result = future.result() + if not isinstance(result, float): + raise ValueError( + "Guardrails can only be used with feedback functions that return a float." + ) + if (self.feedback.higher_is_better and result > self.threshold) or \ + (not self.feedback.higher_is_better and result < self.threshold): + filtered.append(doc) + + # Return only the filtered ones. + return filtered + + @staticmethod + def of_retriever(retriever: VectorStoreRetriever, **kwargs): + """ + Create a new instance of WithFeedbackFilterDocuments based on an existing retriever. + + The new instance will: + + 1. Get relevant documents (like the existing retriever its based on). + 2. Evaluate documents with a specified feedback function. + 3. Filter out documents that do not meet the minimum threshold. + + Args: + retriever: VectorStoreRetriever - the base retriever to use. + + **kwargs: additional keyword arguments. + + Returns: + - WithFeedbackFilterDocuments: a new instance of WithFeedbackFilterDocuments. + """ + return WithFeedbackFilterDocuments(**kwargs, **(model_dump(retriever))) diff --git a/trulens_eval/trulens_eval/guardrails/llama.py b/trulens_eval/trulens_eval/guardrails/llama.py new file mode 100644 index 000000000..fcffad60b --- /dev/null +++ b/trulens_eval/trulens_eval/guardrails/llama.py @@ -0,0 +1,103 @@ +from concurrent.futures import as_completed +from concurrent.futures import wait +from typing import List + +from trulens_eval.feedback import Feedback +from trulens_eval.utils.imports import OptionalImports +from trulens_eval.utils.imports import REQUIREMENT_LLAMA +from trulens_eval.utils.threading import ThreadPoolExecutor + +with OptionalImports(messages=REQUIREMENT_LLAMA): + import llama_index + from llama_index.core.indices.vector_store.base import VectorStoreIndex + from llama_index.core.query_engine.retriever_query_engine import \ + RetrieverQueryEngine + from llama_index.indices.query.schema import QueryBundle + from llama_index.schema import NodeWithScore + +OptionalImports(messages=REQUIREMENT_LLAMA).assert_installed(llama_index) + + +class WithFeedbackFilterNodes(RetrieverQueryEngine): + feedback: Feedback + threshold: float + """ + A BaseQueryEngine that filters documents using a minimum threshold + on a feedback function before returning them. + + Args: + feedback (Feedback): use this feedback function to score each document. + threshold (float): and keep documents only if their feedback value is at least this threshold. + + !!! example "Using TruLens guardrail context filters with Llama-Index" + ```python + from trulens_eval.guardrails.llama import WithFeedbackFilterNodes + + # note: feedback function used for guardrail must only return a score, not also reasons + feedback = ( + Feedback(provider.context_relevance) + .on_input() + .on(context) + ) + + filtered_query_engine = WithFeedbackFilterNodes(query_engine, feedback=feedback, threshold=0.5) + + tru_recorder = TruLlama(filtered_query_engine, + app_id='LlamaIndex_App1_Filtered') + + with tru_recorder as recording: + llm_response = filtered_query_engine.query("What did the author do growing up?") + ``` + """ + + def __init__( + self, query_engine: RetrieverQueryEngine, feedback: Feedback, + threshold: float, *args, **kwargs + ): + self.query_engine = query_engine + self.feedback = feedback + self.threshold = threshold + + def query(self, query: QueryBundle, **kwargs) -> List[NodeWithScore]: + """ + An extended query method that will: + + 1. Query the engine with the given query bundle (like before). + 2. Evaluate nodes with a specified feedback function. + 3. Filter out nodes that do not meet the minimum threshold. + 4. Synthesize with only the filtered nodes. + + Parameters: + query: QueryBundle - the query bundle to search for relevant nodes. + + **kwargs: additional keyword arguments. + + Returns: + List[NodeWithScore]: a list of filtered, relevant nodes. + """ + # Get relevant docs using super class: + nodes = self.query_engine.retrieve(query_bundle=query) + + with ThreadPoolExecutor(max_workers=max(1, len(nodes))) as ex: + future_to_node = { + ex.submit( + lambda node=node: self. + feedback(query, node.node.get_text()) + ): node for node in nodes + } + filtered = [] + for future in as_completed(future_to_node): + node = future_to_node[future] + result = future.result() + if not isinstance(result, float): + raise ValueError( + "Guardrails can only be used with feedback functions that return a float." + ) + if (self.feedback.higher_is_better and result > self.threshold) or \ + (not self.feedback.higher_is_better and result < self.threshold): + filtered.append(node) + + filtered_nodes = list(filtered) + return self.query_engine.synthesize( + query_bundle=query, nodes=filtered_nodes, **kwargs + ) diff --git a/trulens_eval/trulens_eval/tru_chain.py b/trulens_eval/trulens_eval/tru_chain.py index 4d64638fe..7c96b98f9 100644 --- a/trulens_eval/trulens_eval/tru_chain.py +++ b/trulens_eval/trulens_eval/tru_chain.py @@ -12,6 +12,7 @@ from pydantic import Field from trulens_eval import app as mod_app +from trulens_eval.guardrails.langchain import WithFeedbackFilterDocuments from trulens_eval.instruments import ClassFilter from trulens_eval.instruments import Instrument from trulens_eval.schema.feedback import Select @@ -19,7 +20,6 @@ from trulens_eval.utils.imports import OptionalImports from trulens_eval.utils.imports import REQUIREMENT_LANGCHAIN from trulens_eval.utils.json import jsonify -from trulens_eval.utils.langchain import WithFeedbackFilterDocuments from trulens_eval.utils.pyschema import Class from trulens_eval.utils.pyschema import FunctionOrMethod from trulens_eval.utils.python import safe_hasattr @@ -266,8 +266,9 @@ def select_context(cls, app: Optional[Chain] = None) -> Lens: retriever = (Select.RecordCalls + retrievers[0][0]) if hasattr(retriever, "invoke"): - return retriever.invoke.rets - return retriever.get_relevant_documents.rets + return retriever.invoke.rets[:].page_content + if hasattr(retriever, "_get_relevant_documents"): + return retriever._get_relevant_documents.rets[:].page_content def main_input( self, func: Callable, sig: Signature, bindings: BoundArguments diff --git a/trulens_eval/trulens_eval/tru_llama.py b/trulens_eval/trulens_eval/tru_llama.py index d970fa33f..b0235cf65 100644 --- a/trulens_eval/trulens_eval/tru_llama.py +++ b/trulens_eval/trulens_eval/tru_llama.py @@ -80,7 +80,7 @@ # These exist in the bridge but not here so define placeholders. RetrieverComponent = EmptyType - from trulens_eval.utils.llama import WithFeedbackFilterNodes + from trulens_eval.guardrails.llama import WithFeedbackFilterNodes else: # Otherwise llama_index is installed but is old so we have to use older imports. diff --git a/trulens_eval/trulens_eval/utils/display.py b/trulens_eval/trulens_eval/utils/display.py new file mode 100644 index 000000000..dcaa1f0a7 --- /dev/null +++ b/trulens_eval/trulens_eval/utils/display.py @@ -0,0 +1,20 @@ +import pandas as pd + + +def get_feedback_result(tru_record, feedback_name): + feedback_calls = next( + ( + future_result.result() + for feedback_definition, future_result in + tru_record.feedback_and_future_results + if feedback_definition.name == feedback_name + ), None + ) + if feedback_calls is None: + return pd.DataFrame() + feedback_result = [ + { + **call.model_dump()['args'], 'ret': call.model_dump()['ret'] + } for call in feedback_calls.calls + ] + return pd.DataFrame(feedback_result) diff --git a/trulens_eval/trulens_eval/utils/langchain.py b/trulens_eval/trulens_eval/utils/langchain.py index d15bbaed0..197bcae18 100644 --- a/trulens_eval/trulens_eval/utils/langchain.py +++ b/trulens_eval/trulens_eval/utils/langchain.py @@ -1,29 +1,17 @@ """ Utilities for langchain apps. Includes component categories that organize various langchain classes and example classes: - -- `WithFeedbackFilterDocuments`: a `VectorStoreRetriever` that filters retrieved - documents via a threshold on a specified feedback function. """ - -from concurrent.futures import wait -from typing import List, Type +from typing import Type from trulens_eval import app -from trulens_eval.feedback import Feedback -from trulens_eval.utils.containers import first -from trulens_eval.utils.containers import second from trulens_eval.utils.imports import OptionalImports from trulens_eval.utils.imports import REQUIREMENT_LANGCHAIN from trulens_eval.utils.pyschema import Class from trulens_eval.utils.serial import JSON -from trulens_eval.utils.serial import model_dump -from trulens_eval.utils.threading import ThreadPoolExecutor with OptionalImports(messages=REQUIREMENT_LANGCHAIN): import langchain - from langchain.schema import Document - from langchain.vectorstores.base import VectorStoreRetriever class Prompt(app.Prompt, app.LangChainComponent): @@ -84,60 +72,3 @@ def component_of_json(json: JSON) -> app.LangChainComponent: view = constructor_of_class(cls) return view(json) - - -class WithFeedbackFilterDocuments(VectorStoreRetriever): - feedback: Feedback - threshold: float - - def __init__(self, feedback: Feedback, threshold: float, *args, **kwargs): - """ - A VectorStoreRetriever that filters documents using a minimum threshold - on a feedback function before returning them. - - - feedback: Feedback - use this feedback function to score each - document. - - - threshold: float - and keep documents only if their feedback value is - at least this threshold. - """ - - super().__init__( - *args, feedback=feedback, threshold=threshold, **kwargs - ) - - # Signature must match - # langchain.schema.retriever.BaseRetriever._get_relevant_documents . - def _get_relevant_documents(self, query: str, *, - run_manager) -> List[Document]: - # Get relevant docs using super class: - docs = super()._get_relevant_documents(query, run_manager=run_manager) - - # Evaluate the filter on each, in parallel. - ex = ThreadPoolExecutor(max_workers=max(1, len(docs))) - - futures = list( - ( - doc, - ex.submit( - ( - lambda doc, query: self. - feedback(query, doc.page_content) > self.threshold - ), - query=query, - doc=doc - ) - ) for doc in docs - ) - - wait([future for (_, future) in futures]) - - results = list((doc, future.result()) for (doc, future) in futures) - filtered = map(first, filter(second, results)) - - # Return only the filtered ones. - return list(filtered) - - @staticmethod - def of_retriever(retriever: VectorStoreRetriever, **kwargs): - return WithFeedbackFilterDocuments(**kwargs, **(model_dump(retriever))) diff --git a/trulens_eval/trulens_eval/utils/llama.py b/trulens_eval/trulens_eval/utils/llama.py index d89faa3b0..82858de80 100644 --- a/trulens_eval/trulens_eval/utils/llama.py +++ b/trulens_eval/trulens_eval/utils/llama.py @@ -6,23 +6,15 @@ nodes via a threshold on a specified feedback function. """ -from typing import List, Type +from typing import Type from trulens_eval import app -from trulens_eval.feedback import Feedback -from trulens_eval.utils.containers import first -from trulens_eval.utils.containers import second from trulens_eval.utils.imports import OptionalImports from trulens_eval.utils.imports import REQUIREMENT_LLAMA from trulens_eval.utils.pyschema import Class -from trulens_eval.utils.threading import ThreadPoolExecutor with OptionalImports(messages=REQUIREMENT_LLAMA): import llama_index - from llama_index.core.indices.vector_store.retrievers.retriever import \ - VectorIndexRetriever - from llama_index.indices.query.schema import QueryBundle - from llama_index.schema import NodeWithScore OptionalImports(messages=REQUIREMENT_LLAMA).assert_installed(llama_index) @@ -116,65 +108,3 @@ def component_of_json(json: dict) -> app.LlamaIndexComponent: view = constructor_of_class(cls) return view(json) - - -class WithFeedbackFilterNodes(VectorIndexRetriever): - feedback: Feedback - threshold: float - - def __init__(self, feedback: Feedback, threshold: float, *args, **kwargs): - """ - A VectorIndexRetriever that filters documents using a minimum threshold - on a feedback function before returning them. - - - feedback: Feedback - use this feedback function to score each - document. - - - threshold: float - and keep documents only if their feedback value is - at least this threshold. - """ - - super().__init__(*args, **kwargs) - - self.feedback = feedback - self.threshold = threshold - - def _retrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]: - # Get relevant docs using super class: - nodes = super()._retrieve(query_bundle) - - ex = ThreadPoolExecutor(max_workers=max(1, len(nodes))) - - # Evaluate the filter on each, in parallel. - futures = ( - ( - node, - ex.submit( - lambda query, node: self.feedback( - query.query_str, node.node.get_text() - ) > self.threshold, - query=query_bundle, - node=node - ) - ) for node in nodes - ) - - wait([future for (_, future) in futures]) - - results = ((node, future.result()) for (node, future) in futures) - filtered = map(first, filter(second, results)) - - # Return only the filtered ones. - return list(filtered) - - @staticmethod - def of_index_retriever(retriever: VectorIndexRetriever, **kwargs): - return WithFeedbackFilterNodes( - index=retriever._index, - similarty_top_k=retriever._similarity_top_k, - vectore_store_query_mode=retriever._vector_store_query_mode, - filters=retriever._filters, - alpha=retriever._alpha, - doc_ids=retriever._doc_ids, - **kwargs - )