From c09997c7e28b26354d9d377820237aae670f49b8 Mon Sep 17 00:00:00 2001 From: zhaoli Date: Mon, 17 Aug 2026 00:05:51 +0000 Subject: [PATCH] start experiment 18 (exp/18-risk-limit-control-on-the-reference-ense) --- code/MANIFEST.txt | 27 ++ code/tac-qlib/tac_qlib/contrib/__init__.py | 11 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 374 bytes .../tac_qlib/contrib/data/__init__.py | 3 + .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 216 bytes .../data/__pycache__/handler.cpython-312.pyc | Bin 0 -> 10154 bytes .../tac-qlib/tac_qlib/contrib/data/handler.py | 236 ++++++++++++++++++ .../tac_qlib/contrib/model/__init__.py | 4 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 319 bytes .../__pycache__/rank_ensemble.cpython-312.pyc | Bin 0 -> 9461 bytes .../__pycache__/rank_gbdt.cpython-312.pyc | Bin 0 -> 9838 bytes .../tac_qlib/contrib/model/rank_ensemble.py | 189 ++++++++++++++ .../tac_qlib/contrib/model/rank_gbdt.py | 200 +++++++++++++++ .../tac_qlib/contrib/strategy/__init__.py | 3 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 234 bytes .../__pycache__/optimal_stop.cpython-312.pyc | Bin 0 -> 10428 bytes .../tac_qlib/contrib/strategy/optimal_stop.py | 217 ++++++++++++++++ code/tac-qlib/tac_qlib/data/__init__.py | 25 ++ .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 522 bytes .../data/__pycache__/config.cpython-312.pyc | Bin 0 -> 9752 bytes .../__pycache__/providers.cpython-312.pyc | Bin 0 -> 13488 bytes code/tac-qlib/tac_qlib/data/config.py | 175 +++++++++++++ code/tac-qlib/tac_qlib/data/providers.py | 231 +++++++++++++++++ 23 files changed, 1321 insertions(+) create mode 100644 code/MANIFEST.txt create mode 100644 code/tac-qlib/tac_qlib/contrib/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/handler.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py create mode 100644 code/tac-qlib/tac_qlib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/config.py create mode 100644 code/tac-qlib/tac_qlib/data/providers.py diff --git a/code/MANIFEST.txt b/code/MANIFEST.txt new file mode 100644 index 0000000..8d34b3e --- /dev/null +++ b/code/MANIFEST.txt @@ -0,0 +1,27 @@ +# TradeAC custom-qlib-code snapshot (auto-generated) +# parent repo HEAD : 32477c7bb82e90e80b1f5cc2b206bb24c5f95331 +# tac-qlib/tac_qlib/contrib +# tac-qlib/tac_qlib/data +# per-file hashes (git hash-object): + 1b6298c4a5652f2e863cbdc385a1014a570fcd59 tac-qlib/tac_qlib/contrib/__init__.py + b8112569f9b2537c45b6535e1a505a207878d322 tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc + c76a9f17f680e74eea766eff27f7624359749ed6 tac-qlib/tac_qlib/contrib/data/__init__.py + 8d5333ebd2b44165c50cba639ca2d4ac3fc7cfec tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc + 18cb37c0354184c49fa2e598396d7df0634cce0f tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc + 871ff1e163c29261f140c3f53d42a41e6504c779 tac-qlib/tac_qlib/contrib/data/handler.py + b151d139a0dcde87d74b21e7c4b729176ba5c39b tac-qlib/tac_qlib/contrib/model/__init__.py + ab958203f33a99d12c7d923b6efb435189231666 tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc + 9dc36de7e343073b7d511349ee5aede086c38f94 tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc + 9f9014ddd9bce37490061312d51e8e6fe540fec4 tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc + d3f051f3a8650c42fedc7b367b966f7c74fb5789 tac-qlib/tac_qlib/contrib/model/rank_ensemble.py + ccfe7d554989aa7f3e5a2128ae663e51b2207149 tac-qlib/tac_qlib/contrib/model/rank_gbdt.py + 4afcf9058231111c412925f4c4b84e81d656db87 tac-qlib/tac_qlib/contrib/strategy/__init__.py + 74e5ecbbbb20bb71fd5cd083383de4ce88476712 tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc + afaf562aeaa12cebc8529cd916153252e7e3c38a tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc + 79aaad9e39fcc740a773f4f63c512ce1086cfde0 tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py + 92e6e90eb0cd0a25142034560f27adb6b705b1a8 tac-qlib/tac_qlib/data/__init__.py + 0ed1ead6c1314a3f25784d453e54a15a8a04baaa tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc + 9609782800944c45b78bb58eaa7b51ba1b7f8f43 tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc + a85628d71d12cfe5b18b1c884c5d829c89594579 tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc + 686d36f6d101c547491ca866aa143aa542e17518 tac-qlib/tac_qlib/data/config.py + d9f839be30026f337754a3f015425a8efdbe8e2a tac-qlib/tac_qlib/data/providers.py diff --git a/code/tac-qlib/tac_qlib/contrib/__init__.py b/code/tac-qlib/tac_qlib/contrib/__init__.py new file mode 100644 index 0000000..1b6298c --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/__init__.py @@ -0,0 +1,11 @@ +from . import data # noqa: F401 (registers tac_qlib.contrib.data) +from . import model, strategy # noqa: F401 +from .data import TACHandler # noqa: F401 +from .model import RankICLGBModel # noqa: F401 +from .strategy import OptimalStopControl # noqa: F401 + +__all__ = [ + "TACHandler", + "RankICLGBModel", + "OptimalStopControl", +] diff --git a/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8112569f9b2537c45b6535e1a505a207878d322 GIT binary patch literal 374 zcmY+9y-ve06os#U@}ojKbOi=Bf&_g5R0%o&QUQrFd9h3rRRKE*t{FPgmtbe(S$F`X zND%`Q8=@N%*AWcd;pjW}T>IKD-EIf4UWdz@2?2Qb=C9~4ti}kBkU~l{7$OMqOp&Re zh6&bsritN#x6Cy)fe2a-G&7+H3Cti3pNNPSQ4)XPE=WktQ{_}b>%f$0rt5g_ta90G zfl=q`aCD-IRA&}l?@|@Fr=#)l(V3sbPXD5ExlwxJ%4$>=&Xzi1HpF{v2 kv^_lP|1{q~`DBEU2KsNX)xc&0+YRh|alS?$BD;?H4}gGL82|tP literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/__init__.py b/code/tac-qlib/tac_qlib/contrib/data/__init__.py new file mode 100644 index 0000000..c76a9f1 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/__init__.py @@ -0,0 +1,3 @@ +from .handler import TACHandler + +__all__ = ["TACHandler"] diff --git a/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d5333ebd2b44165c50cba639ca2d4ac3fc7cfec GIT binary patch literal 216 zcmX@j%ge<81lR4FvaEpgV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##>w=j?Ny5c_}%mMMca&AwNy#TkIJ~0=L-X;}dgo;^S8`d7ehzB#0^YcoIfC4FrC5ig+@tJvui+AWXvKO%f#Q8u)Xi~c z!WD3lx+Pwna0lE_w+gm+O~MoKBx(b-iMl`?wAo{nP<6*3IErs)@$J&yQ6{iWs21EY zTBs2`LM^<3I%){i3w418VOyY4sJ}r4whLZihtP0?4Kx|3&r(9;4d5&~#t-RREx+#@ zwR*ryD7OP;ljth7?ub%C^Iy_|7SSs@f?j>xw9(hXzi6PSk-q-d_@odQrNbQ0U5m#? zxdC3`i)};aIAx4ixY3DNT#z{ddSp?NxrmfXK#$0cNW38S^>cCls@Q3>jf}-)4*p_S zYj{FdfUJUDD<>uuX_`x?Vo8N-Kbux!sU&y8cbE(q)7F9GB{3pOVlpfWrKxOFDd}n? zo|@vO`9yrL%!NY5<#ZMYhC&>0fU!I*_3Udy{cmvlxwIs9gj0$1gaW)2z5_QH5qV`o z5@k+SBt8}$Q*5RE;6AuCFI}4u6)qBs1CcT&r6!_dFyHy1*eLF1C_E0F6gbfEveKIibf*Io? zFC{lL`+!|69_OTqq-=x3Q&Q;zoSfps>oMp^Mq&G5;0t@32#;|*9@Bu7N+gB4PD!FaB_&`YSV|~#AvFruRoQO4DvD_iPaWA#B*SpbQBml$eS$*dqg6Yf zOr{hb?@;!`Z)a3gA`|gAFGc0F81^wLGX#rLE!sWvag`mJNXJE$y_keCFv42eF#K>- z2+GhoXz&2PV3-Gnqj52Kbqe`@g3|6Ysb7P#WVh4)rKe07mxdS|2>1shU2^}t0s-HYCET0%&BBVwUy67 zHJyaxPV!O>Osv`~&Qh(aJV&)E5)(m3f>Dr`w93Mw#kkKj!?vZ;ViI0sAZRm88wl`> zsVy8&$s#mPPVwm(bK7JpK9LY-S}ur@cCGI~#|yr$%Aei7j_z<#)$#l(GZz^WJwQ`K!|9POBWoJ!_wM50< zFU~SfOtRjty=Tn=oiR^R(k{V3-YpsWZ9~Q|XFg2{#`{I$FVY#)xMSl;{FR|I27$R_ z&}X2j3_I@Jg3ctccMP+Z4g0Ou>GZWrRe~wQz*;OBMldxZe~Vx`if`k%N2e~oGnVqa zH}Cx(8m5BgM+|TprhMj^z`2y9aH$Ar6qZ+P1(y(<=wFHAnRM4Qx2(hrDn)o!Rh2XQQCS(_k zmlvS2NpUI|S833qt3*Z3X-!9^Rr>~RMkK30ff?ke6w z7v4fw(OYz;r`6i6ZbIq=o*nQ{{s|Ow)W=oMxxt@1JXw0x)wr;8+1s(=?ah08mtBWe zy$2t-mwN`5yo2xfvxD=!tDc>AcQ1L`S3DhgPe*p3;A*+slXvZ1^gk5x2Tl~~cjxN6 zAH;Jn``0LgyQWZke%%P~T)ltIM7bMoJ8wB3bmlI6A>R^Qt_~I2e1Gl#3;)Ate*4Mn z>tyyjBg?MVyR#4J{NB$LIyx6;^Bu3u+ZWE{tM?VE++Q1dXK116!>ZjM7whh?o?Y`& z&f2vdRF!*fXdO00ZEZ!8#EMkt5$K?E|qyVa8dlwX)RpTsB0h zVGWEK6U;0EF9X;Tlos;9=q;=O*omTr(cfU-2EY!Pt(4*_jb+DCWH&t-vp!}VE%&BJ z?<;drYIPF5Zw9Q|tR-WbGh{4DCS%H2W?(h5ri@9VZw}B@k_N3eN%o3enDnv36z+Ey zrEJr=wo#IirY=)A`6=qU@iH|<`ts?wFMYtdWwgDD^Fc9((uf(D5$z1zXOc_f}C}c<|)CM*OCW^$a`#QP7 zL|U2VVi7P3<*Cp}q(w;vFOH5M*-Pjz{4>?^bYe6We`Q~1aXG#!32zXLQW0DQg_xw8 z2rQ9RS9+S4q|}s7t(uF&RFk$&m5IhvqpA_mrD{h67nDIa6Fe9LT)Kp}ty%}Khed)J z!ERV_&p|1b0%MZkrK>Q|S^@&9F&v4i7F-Kihar7UH31Avq>_>kSut?Z@B{$^QVw9QE8r+3V&n-!(1S_RjSeTs14MmOSXYb;-4F?zMv5 z^R?se9ACCK7hLYF^&^J+*x{Z#x$5xDzjph~tuwjSo~7EC{=@NdmR>U$t#zxOZS&IY zw{E?4mtU%HU#aiR*LN;?x_-E8`Q=kfFPzSMPG<+eMr`L6cHF96j66{8C9)>fRrj+h z_nM1x_CBqqcC_7md4A7Vj^&vBi#1UE=DD4!ZhS`3R-md~Gs63~GAhFd+j>s48y>e? zPxP>lU!;Ka$~7D$S{D)=?+no z&?NW|UIEOivu+nVqn#YDs}RUHfPD~g2wSEA|H5R1Z3*HN4+;`Li~w0@5(e>{M%gEb z8v!RL$D&C-PC|+Vy4Fyrj0!tp)u9mK3$QfF0T9=gC&y!e?g=8F5;*~n0`xP%jf&vu zVfV>tE(Hw${!3v1BKc0TYeEhhK=?uw1eysNj39r*IYCPpf(BtiLSP<3#~up#4j@&M z0|yw7!6VoTj#v@GSQeuRv@H-cVEB{ln&jg#p-T~Eg(Hw3LYJfhhaBI0 zi<$-SZ*U+eEZ`dGb@ZCI5x7x7sA*5e0KbYRoC1>wzly>|QPb2t+V3+~`28x>$)J)7 z;yD1uXJj!RDfi1T4TTnn=qUh-SQ7A0aK?iIr=j{zy%FVE-VVhaRoHPL%Pd=(3eD~K zZY(tK$TA;V8l^7i4wse#KM?3&YuOKiM=$_#%+M7+WJ_*a^V0pZVEh522mG^!iZz$~^M*AWwnEYa z)Yq6Wc6^PV%q_#sb_RF?KPp&?67b}}H&*HA z0=^Lt0iGrJ$B^zH!;rGUkyDh9^%J)PF`+vmD&eu95DAV>g8&HPb=86Z6D>b0b#T(y$)+oS)#SlVXT{r7C~iVa*|h#V6UXlLWhr) zUc-Ka5K&DUI@bo%VHtrB2&OKYMo`EZXN6S4?=c{eA435GX?J7~-|;M5_-_5(zV9>_ zJoVYv3vN%=RIs~N>^t)I9Xb2HMP;Sy)qK~hxvrr#gW1viQT5<28ut}yx7}eEgq0oL z`5oO4dRAT>$iFzS)G+uY@Z4zFUNdh3pH$__PJZRc!p@)Cn>5z#oqy;28|PB<`Fqa0 z-4FLXa=+*M$IkaUmkys^Iqc6L_J4Tz+;`3|I`hrvHT`gTvUYSGwO()?j^;vR)7MSl zJRHsc_sl|hFVHeZ34hyl;vn_Gkv;u1^SG74@}T|1G2`P`Xec*m73u)ds!LD-t)j-^ zz4?*68G1{Exh1tg|G=QjKI$MuUPixa0(%WM7Gjz6z#gCn01ioppLl%`cA!x)kV}+C zphZTd)vpY!2__nT$C?xyksM(IULM?0^H0I6d+KiY-s-*Ev$CrvzpLlrrTN|^&&gcX zN#gE_aq$`ae(59(^O>bKc&WBvF=B!+s45t|Ho-$;fdob7U{HW4C@7r*I_Yy*d>)E> z6k$Xx9&xawxC}iq0?QlJQ-hhctk)RXmUUY-Tf5$7WNS3T2HmBn7&z1GUx%hY+3eB{ zf)V~OA|tX`6HQy02hwAKl8rdCRD>2xHy~Fk+622`zQO29AFxA9m0+o8sS>OJ5ggF& z09xA(wnPhjuG0Y*T%J?4mD7;CA29?d>1$d#5K@q%eE2H3v{7(m;0X1UqmF|suj&@H7f+Az-7g&qg$~0njAA9owImftv2}$)<#3w(EXBvzhlwPB$O0hb zq)1#RP7j$u2_39v1CF5o)evtHCc+5tTM)-=Od{lmRU7@s2Ktt!>l=Dfwyr(YK6@}Yst~D6^_QVp~nPD z4l2L-$=EYY#*uN}N0R{0tUY6w?Mf+Z6iO}@(n8dDDIKLzW0|SaoiIs9RnNL1xddDy zhD>$F9RW$cW60Pb$-~?MNrl8vO*viljweHX6@om$jL9W%x-}UVf;y{Ux?|3`-lhMP zrU8@qEdH5mg_$i z?QMf-D%jmas&ax4zHo5w6`$%n5BUlddM&l=GpObwR|!!JfFX!pFyW!vAt?<49>nCa z%AkmngpJRv+ACyRwUq^1wdhi<+R^z1<0O5pR&UNZtJRy+&1zK9IU9FlYdyD_ULVNOrAk@(HaYq4Y!d#zycF}V~B${6{B z^WOZ-X=WEK@{Ff!pGp`r3mx(=q4>=lwYG<<+McuTEO_^3{RMk{&fZdJ+LaypRZ~~) z;H!@?rL)u&%vl;%H#dCT=v`^-%{TTwbUtC8#B!H|OO2td^JgyaYEAQ+fp#BWHyS<8 zHOlC4uGy$vdskX}^R2xP*`?NFc}q*yHvc+=h1>Q(v9s%eGuLvwu%mr(f6n{zQ`Xp2 zvu2^V{e_0+rw((y^C@eqw`J|mw^3H-JLbHlVU4m{ZH0zih5DU^Ixkkd9r(9#_fr@2 ztW{I4x^*|yG@nw`Ti9|1Sgry;if(=GWQ9Ed{ZdL601`<>=v=ZLOj+v{Ok|Tsr^+w@ zM%f}5g{z!pA(hO|08N>8k_Le3(%S)M))AOVsAj3;nX=wzD5V5Eb*d6`LNR z1ehyUunSiJ>E}wTSsbVPGFVIPAEtm6?q;Z2Mnp$3~|D)!n8 zoJ_NM`|R020j2zFC?FMVZv-V@t?OKzez<4(#7OSq<)ssWrMfq>Cs(~Ymm#I& zed))0mJUt+c+aErxslK10&nI=-dqk({B&gEq37YXAJsqElRq?>_fEd!Uv<rvFcsCFtnJfhb<}63=uiLK zd+Afv?U&K~-mmWOHN9_!;)4Tq{YRO{h8irlw+zrsxTxDeR`F5l%g}n8s;~%D`7=~W z3jrrYX@QQ?5yS9C+LEOyYkZ^x`K2#aYPW7R1Oo}nKU+DcV8jr1i{*P~VBcp@W}b*h8Y}Pw?Vq%%pj@FtZ)f zP!s92mNh2s5_3oZISz9Wzo3>%e}u&>6e_FWhg``|>hK*aDyr5IfC$9tLzIP}MWthv zF_xr2+f38e8b{_^0s{G;p_rq71({Xx&hRpL)Y``R-i1pGr*ob?xvD*@q(gXjaN)|b zegCSfX8xtG%q*P$@*ll>cu@om7<=Eiyz6M8!MnhJ+cH1%!@wi<5&wbZ-+GqMzwxAF z`K3$m_AIiC{5{L}kLBvea@AwGppX+I`Cv5ng)yyV)l~M@68=LmFHkNr*l=OSDp2_?L7}JJ~)+YIJV?Gp0gbP^qG;``8qA5 zTYKDE)lZo=Xb2uUct(01%_3+Bl-e^)X|W=H2p>GzcX4PWNUp|56Obh1z+&#J8gtYd zqS3Bs!?AAe{7)DtTcDVuR$a|GOS9(8VG*SVp*C4mCsa17E##(3l;!U|x+B*YZ77?1 zNa>YrGD5kHdAoX{_{n1vYMg1Ukc{6$^9$sLP6`CX)xOyH;Nu3*C# zu)wUQ+Mr*}DgVjA3&C?2&h`%u51+j-T<#ef?7QHv=#db8sOEDcm(L9b`=ndQ3o)fy z1rQz5tPrn`r3B)Sh>A!Uq)XpK3P%y6hDA(GfFMjmD-ncOHEcSHq*%;h$Cse^4dw(` z*0NSZ)oq9TPr+SRaMTnUyYX**Q=zW0P}jOrw=Z7@$)pBsaW)ldwiR5SU$-1$ZEKS> zWvY2*aIl_rBh=Qg1|JWx&CI?^7aOtqtd?fIfI3Yz5IVANXkE`qmD~ea7YGAcBYqqp zBBCF%YUmcEZ$Yb%(@qw0BasRGUpbHjL4i{ZDOoi`NCW?qMA5z|n;-~*jJJd)RYLWW zP;Mm@ZPood2|me+og8*H{!XX^1nadgTnhVb6#T^gpbM1Y3=ENFn)ViyvZGJjO)qGstJ?D64s-fM_ z_tnw|o_DsPS1sz0Qf H$VUGk@_{`y literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/handler.py b/code/tac-qlib/tac_qlib/contrib/data/handler.py new file mode 100644 index 0000000..871ff1e --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/handler.py @@ -0,0 +1,236 @@ +"""TACHandler: a qlib DataHandlerLP that builds datasets from the TradeAC lake. + +This is the "custom DataHandler" entry point (Option B): the handler is referenced from the +workflow yaml's ``dataset.handler`` and reads OHLCV + pre-computed ta-lib features straight +from the lake parquet files through ``QLibDataLoader`` + the tac_qlib feature provider. + +The standard qlib processor pipeline (``infer_processors`` / ``learn_processors``) still runs +on top, so existing recipes such as ``DropnaLabel``, ``CSZScoreNorm`` or ``RobustZScoreNorm`` +keep working unchanged. +""" + +from __future__ import annotations + +import os +from inspect import getfullargspec +from typing import List, Optional, Tuple, Union + +from qlib.data.dataset import processor as processor_module +from qlib.data.dataset.handler import DataHandlerLP +from qlib.utils import get_callable_kwargs + +from ...data.config import ( + LakeConfig, + timeframe_for_freq, + NON_FEATURE_COLUMNS, +) + +DEFAULT_INFER_PROCESSORS = [ + {"class": "DropAllNaN", "kwargs": {}}, + {"class": "ProcessInf", "kwargs": {}}, + {"class": "ZScoreNorm", "kwargs": {}}, + {"class": "Fillna", "kwargs": {}}, +] +DEFAULT_LEARN_PROCESSORS = [ + {"class": "DropnaLabel"}, + {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}, +] + +#: always include raw OHLCV; ta-lib columns are discovered from the lake and appended. +RAW_FEATURE_FIELDS = ("$open", "$high", "$low", "$close", "$vwap", "$volume") + +DEFAULT_LABEL = "Ref($close,-2)/Ref($close,-1)-1" + + +def check_transform_proc(proc_l, fit_start_time, fit_end_time): + """Port of ``qlib.contrib.data.handler.check_transform_proc`` (inject fit window into procs).""" + new_l = [] + for p in proc_l: + if not isinstance(p, processor_module.Processor): + klass, pkwargs = get_callable_kwargs(p, processor_module) + args = getfullargspec(klass).args + if "fit_start_time" in args and "fit_end_time" in args: + assert fit_start_time is not None and fit_end_time is not None, ( + "Make sure `fit_start_time` and `fit_end_time` are not None." + ) + pkwargs.update({"fit_start_time": fit_start_time, "fit_end_time": fit_end_time}) + proc_config = {"class": klass.__name__, "kwargs": pkwargs} + if isinstance(p, dict) and "module_path" in p: + proc_config["module_path"] = p["module_path"] + new_l.append(proc_config) + else: + new_l.append(p) + return new_l + + +def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]: + """Discover ta-lib columns present in *every* features parquet file of the lake. + + Returns sorted field names (without the ``$`` prefix). Empty if no features are persisted. + """ + cfg = LakeConfig(lake_root, market) + feat_dir = cfg.features_dir(timeframe) + if not feat_dir.exists(): + return [] + import pyarrow.parquet as pq + + common = None + for p in sorted(feat_dir.glob("symbol=*.parquet")): + try: + cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) + except Exception: # pragma: no cover - skip unreadable files + continue + common = cols if common is None else (common & cols) + if not common: + break + return sorted(common) if common else [] + + +class DropAllNaN(processor_module.Processor): + """Drop feature columns that are all-NaN over the fit window. + + The lake can hold fully-empty indicator columns (e.g. a ta-lib output that was NaN + from the start). Such columns carry no learnable signal and make ``ZScoreNorm.fit`` + warn on empty slices, so we drop them before any other processor runs. The drop set + is fixed on the fit window once (during ``fit``), then applied consistently to every + segment so train/valid/test keep identical feature columns. + """ + + def __init__(self, fit_start_time=None, fit_end_time=None): + self.fit_start_time = fit_start_time + self.fit_end_time = fit_end_time + self.cols_to_drop = [] + + def fit(self, df=None): + if df is None or len(df) == 0: + return self + window = df + if self.fit_start_time is not None and self.fit_end_time is not None: + try: + from qlib.data.dataset.utils import fetch_df_by_index + + window = fetch_df_by_index( + df, slice(self.fit_start_time, self.fit_end_time), level="datetime" + ) + except Exception: # pragma: no cover - defensive + window = df + if len(window) == 0: + return self + self.cols_to_drop = [c for c in window.columns if window[c].isna().all()] + return self + + def __call__(self, df): + if self.cols_to_drop: + return df.drop(columns=self.cols_to_drop, errors="ignore") + return df + + +class TACHandler(DataHandlerLP): + """DataHandlerLP backed by the TradeAC parquet lake. + + Parameters mirror ``Alpha158``: ``instruments``/``start_time``/``end_time``/``freq`` define + the queried window; ``feature_fields`` selects the features (default: raw OHLCV + all common + ta-lib columns found in the lake); ``label`` is a qlib expression for the target. + """ + + def __init__( + self, + instruments="all", + start_time=None, + end_time=None, + freq="day", + infer_processors=DEFAULT_INFER_PROCESSORS, + learn_processors=DEFAULT_LEARN_PROCESSORS, + fit_start_time=None, + fit_end_time=None, + process_type=DataHandlerLP.PTYPE_A, + filter_pipe=None, + feature_fields=None, + label=DEFAULT_LABEL, + lake_root=None, + market="US", + **kwargs, + ): + # default the processor fit window to the queried window (like Alpha158 without a split) + if fit_start_time is None: + fit_start_time = start_time + if fit_end_time is None: + fit_end_time = end_time + + infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time) + learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time) + + feature_fields = self._normalize_feature_fields(feature_fields, freq, lake_root, market) + if not feature_fields: + raise ValueError( + "no feature fields available for the lake; set `feature_fields` explicitly " + "(e.g. ['$close', '$rsi_14', '$sma_20'])" + ) + + label_expr, label_names = self._normalize_label(label) + + data_loader = { + "class": "QlibDataLoader", + "kwargs": { + "config": { + "feature": (feature_fields, feature_fields), + "label": (label_expr, label_names), + }, + "filter_pipe": filter_pipe, + "freq": freq, + }, + } + super().__init__( + instruments=instruments, + start_time=start_time, + end_time=end_time, + data_loader=data_loader, + infer_processors=infer_processors, + learn_processors=learn_processors, + process_type=process_type, + **kwargs, + ) + + # ------------------------------------------------------------------ config + @staticmethod + def _normalize_feature_fields(feature_fields, freq, lake_root, market) -> List[str]: + if feature_fields is None: + common = get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + feature_fields = list(RAW_FEATURE_FIELDS) + ["$" + f for f in common if "$" + f not in RAW_FEATURE_FIELDS] + elif isinstance(feature_fields, str): + feature_fields = [f.strip() for f in feature_fields.split(",") if f.strip()] + fields = [f if f.startswith("$") else "$" + f for f in feature_fields] + # de-dup while preserving order + seen, out = set(), [] + for f in fields: + if f not in seen: + seen.add(f) + out.append(f) + return out + + @staticmethod + def _normalize_label(label) -> Tuple[List[str], List[str]]: + if isinstance(label, str): + return [label], ["LABEL0"] + if isinstance(label, (list, tuple)): + if len(label) == 2 and isinstance(label[0], str): + return [label[0]], list(label[1]) if isinstance(label[1], (list, tuple)) else [label[1]] + return list(label), ["LABEL%d" % i for i in range(len(label))] + raise TypeError(f"unsupported label config: {label!r}") + + # ------------------------------------------------------------------ utils + def get_label_config(self): + return DEFAULT_LABEL + + @staticmethod + def discover_feature_fields(lake_root=None, market="US", freq="day") -> List[str]: + return get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + + +__all__ = ["TACHandler", "DropAllNaN", "get_common_feature_fields"] + + +# Make `DropAllNaN` resolvable by bare name from processor configs (e.g. the default +# ``infer_processors`` and workflow yamls that reference it without a ``module_path``), +# mirroring how qlib registers its own processors in ``qlib.data.dataset.processor``. +processor_module.DropAllNaN = DropAllNaN diff --git a/code/tac-qlib/tac_qlib/contrib/model/__init__.py b/code/tac-qlib/tac_qlib/contrib/model/__init__.py new file mode 100644 index 0000000..b151d13 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/__init__.py @@ -0,0 +1,4 @@ +from .rank_ensemble import RankICEnsembleLGBModel # noqa: F401 +from .rank_gbdt import RankICLGBModel, rankic_feval # noqa: F401 + +__all__ = ["RankICLGBModel", "rankic_feval", "RankICEnsembleLGBModel"] diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab958203f33a99d12c7d923b6efb435189231666 GIT binary patch literal 319 zcmX@j%ge<81lR4FvQ7c%#~=<2FhLogHGqui3@HpLj5!Rsj8Tk?AU0DDQ!aB9b1q91 z3nN21LkjaE#wgZG7ERWdj6mg@jJL#s67#Y>on7;aQ*)DYQhnT=eDhOMb2OQ5@j+x@ zBDZ*ofGRVSqzEXG3v!Zve0*kJW=VX! uUP0w84jZ6YX-=wL5f9J+kY9=gfW!x8Mn=ZF42Jg^j305iHL@3h!VLf*)l?q< literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9dc36de7e343073b7d511349ee5aede086c38f94 GIT binary patch literal 9461 zcmb_idvFv-dY{?ZmsY!af+WDe0K(D=Y4rdBmJOVESlGO5gYB|xZZ+B&Nh@Y|#muaQ zl+_(^QN=FD9iq-Imf(wVC6$wdtBzFVf%A`jRhPP}O8$@#j<)8wiYvZL#s3M*7yDeP zN`7C@&aNZ`m83F)o}TIMue-nJufN%U3ED?e*sSl=S-0$E}ve{Jnnm&Z7om zWxZvr?p4cU6}=Uxmkjc~l@6|llYJNPq*UAztQGWDjS0chs z;?d~fkkPaMm}m@1hNwuII@Y2ai4k2)#Knk~(DfEwi5SsDTvElCr1+WV4upKZzCNpZ zC{O1oC?sS>?duaqH3@_54vQKZM-;6^md5f_!~x|^klY4ZF~!iLkv$E^6LF;hB(`Dr zKvdI>u+e!KJ@2JZMBBGw`Z;8=@#HcPBniP$r z3+A`_jz*0kF)Ax@BO1XHhQ?@RBa$XzIz`i)MJX=(Ml?l^vW@Gaq$#3=$)rJ$kr-DP zB|Rjb8o~@T$&eyiGe%TNXV2J~Aw^m=xkUrt3OsL#>Iqdc6d60xlmSIk;t@sEl5ue~ zswp5AHxgph(8bY&c4k0LjQSeEqJ*4OmGFpU40Va18a}H=`$LgL9GvS9v57-8OW5W^ zUtbV%I)zo)Z$NlR(TFgu)so*LffGx`eq~5{GnzW1=-X zV|9bRe67R_+9FAp)(#<%3{_Pjx-!^Ai=t@@+JJ>pZ*T%?V+?7E1g%IURPkTWzYS@r zgZ&}a0g?f=1KYNdCO}B4BI)^+_B?mAS=19ESRYAhkey*^f+#1%c)}3-)kNeBY{y5l zqqEr9neXtu9c{5_ytR`I%OI`kPn-*e#ABeHgs#zRN!-~oGBl=>TDC+I(7IN!y+b^C z?38aPp&P|5>7tGbbIoj86&g;qOFLm3hdx<)8&hHlZLFmqI~k42Q2hQx66#GG zQ>4g{xIY2Pu#5Y;9%A2-^T>u=&nK+xPWb1axp9-e54J1jF?R z6jOOxR3&W?*2OkMd$3pDuNat>>_-tvmql4JB;QDM1k!^6gFT~?7GrFIbugAGSfh&8 zL0fc<15s7Aly3<8QQ0mcLrT!HV75InBD{w*;v%#R8iM`MHk+Cfqw%ID?2*KN5VPxw zf$cR$iGyoe(i2cc^DnlcP9c;i$Agn21os1oTBa#Z(XcY;yCCj^9c>*uo1c2BxxHii<|V7L z{dAYu>GO5N(f9Q=HCa+6TY@5zHcD~4X9WBDTKoF!LFhpWJ&@3#1~RlW8IPbZX}6K^ zDG1X<&x&u0;71_{h>hR~>;xWSfQ2nswpN_tO7iCM=X4Y8G%MB%B zK{u;(S~q?{Mdc!}q{8*`z>^M{2Y&R(-ZVD=WO*Ub>r`BFi7Z@j-pY^fb>rE2!P)C6 z;p{)zXV4s_Q=F6b|J3BViNoqNe2*#Jdc9-0XZ#&bvBj@vC+0_ zfB-DIAKHV(StQ|Ct(ad7*y>mavTlPOHOk6>lvItNxJQ&AOh$|r+lWA85oosJC6>}Z z#yq6-rW92rU}vlqu_X93XtjgZ9`rw(Bupu()~alk#;S_SiuL?YRFrwt!g7`)R0N5~ zSO!I=B!y0x5pXL4J#@H0tFRPnkr5%UCUo=Qe<{cD3qz3hJNz}yz}>o1{H=U_c=3pQ zt6XS3$X#(p9D`iMak`n~CY&kfaQ+lzQ=+@An>v4al|i$z>+yZ<`L8-Ihn%1f+FUqq z7aW)3hP^hO;$1H_upjG^9X5rx_(gLT-h!Ij?rW6WwUl6wLVIBbquOrIxKi8?8#zuE zqO^tz`+f2L=lqX&o*P7KxUiNnteMqSJVNGwyLd0VY>5Uc-<@shgp;lhjn5a*^GGp+s;;L zWTcf2J%nMX!p(91RyK31w)mV98X42pfWr93mAzodUf4_Mq-RikbDp~&YQMep z*6>_t?@Va#9sOSD!0hT5XF><2DnB{<%h8{Y{=?YYk?f}T8m>3o2)+}a4jq_T{bJgA z*_SJ?zA}Dk{C?FF)9as|t?HTf_uQ{rmpyzVfY!io*KfM{S0{1M*i0vd>UQ#BPh;u^G>0p z^?r5D&-Vpch|N| zb^EMpb`Jhd$tXBl`9wIHvp!%ZiC zh?b-RVe0^8$#J~?Q36}sOC}ooQJm*;rR7)lWcFm+@0LCR@x85Qcf8kiz3a!{pK|^z z@Ihdp&Z!KJ*z{=z~El?*??XI5pZGtQJN93!kc3V7(&{vy_N1%RtJ15}?lhNnrfUXD07|_RK7-iEu1w8`9^0lmxrtJP z6qjA8(&6=Xy?9UgR+fxCt4(XdFT3c^e$R8?_XE;+uLL3j12u5EZ$R@Vyc2jyEr2bVaIcm^|UKD^(FK@L_n$w8WG}R;epv$^We2hH2KN)d7JIL$%4hJ7z}4lG`2Sre z$9Ax7@eaVnhKH%8%2MUB4ytxOtWuVSvzI(ASxSCrUclM<8pG@4Bm*%(SiArVy| z_fR^jaGF}4h9t^Ma+Ww19766#FVI?^_bKjXX~IWrT#|A^x5WnrivcW4g_N?jX;IWj zJqaNyaz3n><&;fXloK&mub2Wc{WzlcCv+uD9{@~0m3kO(00%MCrzxWfeUebL<4fW) zc{%tSe&OFyd8cIY(FFF&r^vhdhgb0E%)9cLGTFgm_Ig=J@x#uCb@)wQE6?~=Zj%MF z473$|ZE<|Yohh{~Xu<(6&aZS3Njc!fOETU}03KYhJUH_s!i00emBO0B1<01-Q}Fcz z0uJBq<5~z)Sxhl-g z!$|rhV(bmlp1SE`&^%bqrZf2%4hwo%H9h&{uIXY@GF@?;;dRqP2`h}#o}h`Gaiq#j zPuxmznWZ>SSZbhX+ICtF`DV6A(_`TToxw0mmCoCj1vtC|IE9c_jW<66YoJ5u8D!a^ zHG4yL?`xPse;dU&I2}}Mn5h7en6B72b#ShE?+pG`>`Qm&%4@Qfm;NX_al8BP4*%8R znT^lnnmQ->Yu>Bg?E0CChFtZkYx}P5ySZ!Xd0@))!CZaU&F;DSu9^C-T*az+!M(NY zGqEA}WK(W^@IeWl=6ziCng^w*&G_Dka^JLnGea>* z{FDMg=kb(2?h4T^O+JJMHzYI=J!i>q)gsA~SGVw4QLX~^tgWSnT~r_taN1@n2*PQk z*``yagcJZfj)g$zrVF2Bpb0*RET8evc2YlI7-?ZgC9_qE1TKF@ja7Gz31=zUu9eUkO#eOOdpwLNEkvtLm z;L2>*g8O%HF@4Yuqr>>xN2>zCbA18f;lCjH`FgH;^WCbaZl0Rk`qa$Ur_!EWdHvn; z{Zr1l#{Dym`_sSwCzXRDq}d%kjWwQIARvZvmA{rc-S zPEQTatnZwy*_kf=+8HR>@)bwL2Cht;>iIOV-4cTv_J>H`mmDueSYz+UtW^>1O54?mywDYVX#zC!E~-qLin ze3GB+%x?YlnhlfPx%%KwpL_qgDd~3AZ2b<)7XG>Znd{!HkUg2}J)QN>)wRylwcc*O zz2)}HciKPR{n74Eo}GF6$aL4y>9%9P(q`*knm+l;-MUvF2;9~^DAsPA7r6Ru3tmoa z$ec)br<0RP&R=omMCL^H(9O=; z#ZIOJJN|RQ&DC}2RS^C!oKN;_5`M8O(9`1n7QYdF9NEh$)^{0t}Ugp&X~qaYs$NrYnxzhxbVSpbJrCU#_|?TX}VN`o#T;s%s@zOR`&UO7|)n)2>|k zD)KMcv+42K@=ep;O@H{ZgsVKt>lEMrVr}ole8-`@$h=7)9~T1@3{v=koqX zo!8YiU&FbpzH~TUtqTHb^Q`6Ov{2fBlNX>)Jfep|C=9wKj+)5CaC1U_Th3kzdg2>1`D$ zsrAV}WHswnv~?R}^xZ=v0c(^oAm|mBC0*;Khe`t1Awo>7AlO9WGuJ%yK9oOWXgFkY z=3{^6G1xB~4C^6Zi|=&cE@LA@5Nt1bH1&)n$-SBelEirZX%xtX@%#fv2k-nnGX4CP x&$*J{aP9xXZTg(6{vG$^Z@BHhkunPbH literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f9014ddd9bce37490061312d51e8e6fe540fec4 GIT binary patch literal 9838 zcmai4Yit|Wm7d}IsV8O03hj*Jmnch=ZAFeP$4(tft`plz9J?>Y4mIKoDN=lSW@uYl z3Q&UpmAh^uwdqRPq^N>sH%c8KE?S^oV1ez=?rwplA~)hrRlwMGvB*Dpo$eO-wcokJ zAsI&Pj;)zH_i^sI=Y7ubU;Ta$him=htI2yiIqvuLVqSKm!9RK%jYUr4MmdR>EGa&1 z8MW}#wx+CU+o&yVAGN0)qYl<*OF2hf7_+6^qaOD3j{5Mlr~K)L(FQihk!nmgjW)5i zGu51K8Es*0SE@A~7!9E9mOOFGXq(g^d0(@QZj*ekaii@P?#rCye+{E@%N?tM-Dt;@ zEz~G})4_4)zx?cltR$y|>!OknGn$}Hh?*daN@}WC)v`HN$YzA7l2z4SRgP+jY(`88 zFNvAS=S~ZWjH=0^BxGYk>TuZOxul61NmL}^S}HLvjE$MF$HwXbY7?@c$ZaM^1wBrO$j$-L5ym7F_oGUqN0Mg@oZib z62`Qgtn^Calt%%=LNu!=a!RB#2wijp;knbH9^uABB03>p^BN>`iA-FOva%{CI*gGHz0|UJ=S=92tC?{s* zR7g-KvN!6fOZ#|$jwnR)pmy4%Y)rlmGN%$T>`^B6!SKN+pAvdc3TQkvAY50&j1a(^ z@hK-uLYf!}K3tx_@uMjbL{;Rxic{3`V!tX#q9zJCMb3$eL5}gfHt6vPLa(5SaTSCB zMaITb@$v8wrmC_A7T!o`6GEAwpuen4VM<)d=5rWSK^|35vp5OSnvNylO3Rv(h=MU0 zVO$0S(z(0_L;*$66j5SS7N?H0ofDZP1Wc{cVr-0&0noeBG9Z|HU&=u|)98TQ;SdsN z0n(2XjT0Fy%QiNU#~z8y*w}G{vjWL8<1BcV5mY(F_9F%+G`UO(CjM#hCS$nDV3ipF zHqlftCLv+5W?&7Gh%y2jM?JBBB%6`<1Lp&vPAs9QTErNMWHYI$6PJ~|OytVe2^-r8 z=td_i45#@*7s(%OTc<4{u=#5VIy@a2G`o>iCSxf`*p!$i$(fWHL%{+Pia`dSIKsqY zuq;%jf>uT`IiHdvIZ>M!geXKKq{-oEHUl<}hYk56{zl^Cl2%zYc|%mBm54@|fJCi^ zH)MA=?s(@P{lr3kyI}cdgT|ZV1vmBT{DgXS8%X#-gwTqW}G!UI8DfsZ4ztLgEgmYGGNr+3)T6Ia%#u_Gv8H#U#9eJ zv0zKK*1|N}HE;r3jJBJ7$qutseO9qbK345DTX)J0?JvWqVEUI}m~@pG&CRp%-0ff6 zxPndDsqHX#tvL^3%KEEdGuN8WOPpl6#DyHwA79wgIWj|78}gLBrCM>9Dl7~c<*FB- z$fhJCaDpKohm*z{C3^#LPRxi2YMi#vRcj`CU~3q#tOCOj7KUI!jfmmt{<6z704eZQ zumv!&h&9Q3_c7N^CPh#u6FI~&VFpMciHw_c_=NQD6H>X5g^ef;)NazPDLJECGCAD~ zs8LwB4E}USG@n-UY26{J+Eh-~t?-h%9VCg%x?N2rqUcgJ3FFu>Bk3+8R5UBcbW0As zmtfmuRkslhbvxXusOdI}K|*fbPDrb|gQ2bJc4iWFTQ(NM0$5R1=My@Qdrs$5DycZp zm|dS8>cf70@T`}S_^CC{sHyO#Id@3>d?e%SMV&(GDh z?&E91)5XpcYu=NyLnW_&c4))j@<#uB{`-R;`*+Uq8@}fGJ@eNV_AfoT%rBpM$NuK= zHQ)X@zSJ0)zj*8B(za#(&FyoCFwxsEckQo_%%A%9;M*-f+V&;}-rA1scYbjA!z1q> zxx4Gmv3vYaPL_OulFRqH_m=mKXBW>coGYWV>&L!z;n1pZXiez<`Afg}#!tU-@7m9< ztPh-79XPW#aCR;5+^_bm2447$>%~nk*YNm5AGfFL_P`r0i|q^TOM&n2So1z!v>jZz zgxe#Kf{T{z7qvOc!I&+Np#fXYCo7hmhK68(EAowA@d}qv$(Ye(V#Zh>vMT~`)1C0=mi83AJBzlR$^op7;;(X*Js9BtNoPJ6g_~vBqkfT2hnbzzpnhlOJ`12hEBX zXiPt2x_iV@U(99AP^Qe7>j_b)WM3>S2!BDA0F{4q7@&W^FLG61c*|Pgz5@zKd;)Uu zF8^JghpZL2NJSQ>V68^phj75?iFHHK`u4Y^26c?vt|4kMV> z-H`|j)guv=hzy3#8vHy$M{GB+HxC*%!G0BkYA0@gn&mcJ{#(vfSI4UBiKQbSxt=I( ze`3z|od$!6wF2+Le7u8+xG%!y&R8MXd=lOD*L4`3=ebwG(jji1zxsCw7;FXGpPkF! zZrxhBpf*;?7Pn|+0phsi79r9H_KK?63-%=8S$8R3jabeu$zi(2S9uC;SP)~EZ1kLQ zBCv5LNf+v_B>9oLOLDRx@996YVv8_Ga?vi5i>(v1O1P5TA~e|=s)_rIGgEsex7m*~ zS!S&gH{;5Ls)0$tB6-XZLuo-sNodqv1-l6YYinV9&3?e}!&5>-EV!bxZJiol~B{?EPOHk$fNcO`MZt-0H47<@e-)x&gduEoNV` z$81%fA2gV7XWSVJsO;AI%+cgQvsHZ-99zj!!C7z>+*0EQO?S#JW5!dkCi|=F%vQlu zu*E6?Jr7;Fy@G&{97*$tZfA811UgS+zHWz&NJzS~Tu##MMyjVq*Ii4mWRrjS@;vOS}dxX?yYz-DuHG8 zYXYoSE~_BzMedvEH6k{KByxG(rN}pA@_p06uVl(Ikrs&#utOari!`Q`B`m~d3UbeEQmAaq9DhILy1VT7mgaosso z_SB@EA+JKgn?h2hd#XgzZIT*E=~l2&w@5Lazp3+6I)B3;1uv;&=->v-i+KaxA1M=p zk!U(t?-y!EM*%>cW-()_v3br`YTUN8f4#GN75^K%i>~fsWA{eu-nr8VAm`)j+xD$) z+xJo9zEWFoF|rU@zI6B4y?F8Pnfn)CS-%)vy%?R3thGt=mQrim;?eIN-RRs?+$*ki zju!*trS|Php@o^{@s;*>Cg%^$^QG3n;?aepizgONET8(Qb??fq54+y)`Y`-{_+ImB z_wjqzKI;C`Mn~t5Ja2lI550Hvoug|VJqxG)Kp-!#b$+E7_{xLU9mUqYrH9x+GVqj>~!38=NPk!&@yUDvPzv%dB$3Gta zrxX8h;{Lwpew8d_7N1Oc4oABCZRh*j;gmIUSNm58;P z7Z&*MR;mJ5FdI(IRgF!za-aq4tPiGr3&)w^5yzJIsDa@DHg#9Q_Cdu^)y&+&l~x^1 z+&bgNzFW9fWv_&&DY=ykwUM=x?0;s|L8Db8KI6#@Zap*ER(Ak`ib&mbMu^e2+<(h; zj@oq+KjW#Pz_u^$vo+*`Q*x5#$1HH+#E&g165xl0xepH~C?v*WE2>%{B08rDqW-KdyQpgTNXYFwnU}e`E=Y19ZFVLJ`?^cANLJHa7u*T{U3?fX zxadhK_U*$q?)C`&kQh(dL(b`A=d*Fss7Orv4K=|xRRv#?$XJx^555%B_Z`y!g>@eP zzohdKgm0Rh8&RGFCLxDzMezj@oF=DL-7Q1o>FXfu8b)<*jXBhv_=cJiqjJb&XctK^ zQx_N(MUEv9!%Zp?T4sgzA`C^)h&VNujZWx%S|xy_IL2kDk0)X(yE?tSUTG>rcd$~O z#%-4See<@tQ>9?nda!3T*t2r+u4OIQzaBic8a(!K@c4pj-a3D&)F~`=Ebm@!TNYRN zWwqFTVBWSNgjRO{eZ#`p`QfEQ8-cdP3kw%k1Lv17zjx)GD|dI_RoC_&zjv{?=NWYV zQobKJ|Io_q>U(JCc07$%OS>V66&3>*iZj4Y{?Xr{QH^Eb@hZ^_2Fe#IB@UCxGcF5x zWy(P6u7WE@p3#aTgI(g^wiTQt1Xm#LwRM&mJBv4JaVf(o*b8oR7oKDF7^E7L7-jZP zxk)YJ8hT}F7jV<c zp!k_5lMB;NdlYfXC^!EQy(}|TSc~|mP<<%0U3m)+qG2pgzkUI+psWxlr1>ZoDCD>l z@*E0fK}e%1TQ*X12fkxVqH5&Yt`t=i;^S!}P--R%9;Pw?yHVMjN$K`4^oJ*m&%s7z zpq-7Ezps>kz(BuUP$=)x{cP0y>4ah5!D9q$K)aieq!5R(i@63y6T?iqo;GCUq_To3 z#9;LTZktw~=O0@3^0q&Y@EpJUzqqbduIu+)%Wt{X-*VgjliT+@uK#yj&+oX$9(W&H jdZp;;nRPyLH1UW2d!L?X2*~q#l literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py new file mode 100644 index 0000000..d3f051f --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py @@ -0,0 +1,189 @@ +"""Seed-ensembled LightGBM that early-stops on cross-sectional RankIC. + +``RankICEnsembleLGBModel`` wraps ``RankICLGBModel`` (per-day RankIC feval + +``metric='None'`` + ``first_metric_only`` early stopping) over a seed ensemble: +one sub-model is trained per seed with identical hyper-parameters, and +predictions are averaged across seeds. This is the model class the +``tac-rd-rank-ensemble-isolated`` reference run wires into its workflow +(``module_path: tac_qlib.contrib.model.rank_ensemble``). + +The ensemble inherits the RankIC early-stopping behaviour of the single-seed +model (valid RankIC drives the stopping iteration) while the seed averaging +stabilizes the prediction against any single seed's early-stopping path. + +Training is parallelized: the seed sub-models train in a thread pool — +``lgb.train`` is C++ and releases the GIL, so concurrent seeds do not block on +the GIL (5 seeds ~40min/5 on this box). Measured on a 6-physical-core / 12 SMT +host: the seeds scale ~2x, not linearly — the runs are memory-bandwidth bound +and each Booster caps its threads at ``cores // workers`` so 5 concurrent +boosters don't oversubscribe; larger-core hosts scale better. The qlib data +pipeline is warmed once on the calling thread (fills the handler cache), and +each worker then prepares its **own** ``lgb.Dataset`` (independent handle, so +no concurrent ``construct()`` on a shared handle — LightGBM's ``Dataset`` is +not thread-safe to build). qlib's ``R`` recorder is also not thread-safe, so +the per-seed evaluation curves are logged on the calling thread after the pool +finishes. + +Wired into a workflow yaml like: + + model: + class: RankICEnsembleLGBModel + module_path: tac_qlib.contrib.model.rank_ensemble + kwargs: + loss: mse + learning_rate: 0.02 + num_leaves: 31 + n_estimators: 3000 + num_boost_round: 3000 + early_stopping_rounds: 200 + min_data_in_leaf: 20 + lambda_l2: 0.5 + colsample_bytree: 0.8 + subsample: 0.8 + subsample_freq: 1 + reg_alpha: 0.1 + reg_lambda: 1.0 + seeds: "42,7,2026,99,123" + parallel: 5 + +Any ``**kwargs`` other than ``seeds``/``parallel`` are forwarded unchanged to +every ``RankICLGBModel`` sub-model (same params, different ``seed``). +""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from typing import List, Optional + +import pandas as pd + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP + +from tac_qlib.contrib.model.rank_gbdt import RankICLGBModel + +__all__ = ["RankICEnsembleLGBModel"] + + +class RankICEnsembleLGBModel(RankICLGBModel): + """Seed ensemble of RankIC-early-stopping LightGBM models. + + Parameters + ---------- + seeds : comma-separated integers, one sub-model per seed. + parallel : number of seeds to train concurrently. ``0`` (default) = auto + (all seeds, bounded by the available cores); ``1`` = sequential. + **kwargs : forwarded to every ``RankICLGBModel`` sub-model (model + hyper-parameters). ``seeds``/``parallel`` are consumed here and not + forwarded. + """ + + def __init__(self, seeds: str = "42", parallel: int = 0, **kwargs): + self.seeds = [int(s.strip()) for s in str(seeds).split(",") if s.strip()] + if not self.seeds: + raise ValueError("seeds must contain at least one integer") + self.parallel = int(parallel) + # drop seed/parallel handling from the base kwargs, keep everything else + self._model_kwargs = dict(kwargs) + super().__init__(**self._model_kwargs) + self._models: List[RankICLGBModel] = [] + + # --------------------------------------------------------------- helpers + @staticmethod + def _cores() -> int: + try: + return max(1, len(os.sched_getaffinity(0))) + except AttributeError: + return max(1, os.cpu_count() or 1) + + def _worker_count(self) -> int: + if self.parallel > 0: + return min(len(self.seeds), self.parallel) + return min(len(self.seeds), self._cores()) + + # ------------------------------------------------------------------ fit + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + """Train one RankICLGBModel per seed and keep them for prediction. + + The qlib data pipeline is warmed once on this thread (handler cache), + then each seed sub-model trains in a parallel worker thread on its own + ``lgb.Dataset`` (LightGBM releases the GIL in ``lgb.train``). Evals + are logged on this thread after the pool (qlib's ``R`` is not + thread-safe). + """ + n_round = num_boost_round or self.num_boost_round + n_es = early_stopping_rounds or self.early_stopping_rounds + + if len(self.seeds) == 1: + m = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + m.fit( + dataset, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + evals_result=evals_result, + reweighter=reweighter, + **kwargs, + ) + self._models = [m] + return + + # Warm the qlib handler cache once on this thread so the workers' + # concurrent prepare() calls only hit cached frames (no first-write race). + proto = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + proto._prepare_data(dataset, reweighter) + + workers = self._worker_count() + # Cap per-Booster threads so concurrent seeds don't oversubscribe + # (LightGBM's num_threads=0 uses ALL cores per Booster). + per_booster = max(1, self._cores() // workers) + + def fit_seed(seed): + m = RankICLGBModel(seed=seed, **self._model_kwargs) + if workers > 1 and "num_threads" not in m.params: + m.params["num_threads"] = per_booster + ds_l = m._prepare_data(dataset, reweighter) + booster, evals, names = m._train_from_datasets( + ds_l, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + **kwargs, + ) + m.model = booster + return m, evals, names + + with ThreadPoolExecutor(max_workers=workers) as ex: + results = list(ex.map(fit_seed, self.seeds)) + + self._models = [m for m, _, _ in results] + + # Merge + log evals on the main thread (qlib's R is not thread-safe). + if evals_result is not None: + for m, evals, names in results: + for k in names: + for key, val in evals.get(k, {}).items(): + evals_result.setdefault(f"{k}.seed{m.params['seed']}", {})[key] = val + for m, evals, names in results: + self._log_evals(evals, names, prefix=f"seed{m.params['seed']}.") + + # -------------------------------------------------------------- predict + def predict(self, dataset: DatasetH, segment="test") -> pd.Series: + """Average the per-seed predictions over the given segment.""" + if not self._models: + raise ValueError("model is not fitted yet!") + preds = [m.predict(dataset, segment=segment) for m in self._models] + if len(preds) == 1: + return preds[0] + frame = pd.concat(preds, axis=1) + return frame.mean(axis=1) diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py new file mode 100644 index 0000000..ccfe7d5 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py @@ -0,0 +1,200 @@ +"""LGBModel variant that early-stops on cross-sectional RankIC instead of l2. + +Standard qlib ``LGBModel`` early-stops on the regression loss (mse). For +cross-sectional alpha signals the quantity we actually care about is the per-day +rank correlation (Rank IC), which mse early-stopping does not optimize for. +Experiments on the 50-ETF lake (SP-5d 55-feature panel) show that early-stopping +on a custom RankIC feval lifts RankIC 0.047 -> 0.075 vs. the mse-stopped model. + +This class reuses ``LGBModel``'s data preparation but: + + - tags each ``lgb.Dataset`` with per-day query ``group`` sizes so a ranking + metric can be computed per trading day; + - injects a custom ``feval`` (mean per-day Spearman of pred vs label) into + ``lgb.train``; early stopping then selects the iteration that maximizes + RankIC on the valid set; + - forces ``metric='None'`` + ``first_metric_only=True`` so early-stopping + tracks RankIC only (not the regression loss). + +Wired into a workflow yaml like: + + model: + class: RankICLGBModel + module_path: tac_qlib.contrib.model.rank_gbdt + kwargs: + loss: mse + learning_rate: 0.03 + num_leaves: 31 + n_estimators: 500 + ... + +The rank feval is used for early-stopping selection only; the objective stays +the configured loss (default mse). Set ``rank_eval=False`` to fall back to the +plain LGBModel behaviour (early-stop on the loss). + +Generic: works for any cross-sectional panel whose qlib dataset index has a +``datetime`` level (each level value = one query group). The per-day groups are +derived automatically, so no universe-specific configuration is needed. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +import numpy as np +import pandas as pd +import lightgbm as lgb + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP +from qlib.contrib.model.gbdt import LGBModel +from qlib.workflow import R + +__all__ = ["RankICLGBModel", "rankic_feval"] + + +def _per_day_spearman(preds: np.ndarray, labels: np.ndarray, group: np.ndarray) -> float: + """Mean per-day Spearman rank correlation of preds vs labels. + + ``group`` holds the number of rows of each trading day (query group), in + order. Days with <3 valid rows or a constant pred/label are skipped. + """ + if group is None or len(group) == 0: + return 0.0 + offs = np.concatenate([[0], np.cumsum(group.astype(int))]) + vals = [] + for i in range(len(group)): + s = slice(offs[i], offs[i + 1]) + p, l = preds[s], labels[s] + if len(p) < 3 or np.std(p) == 0 or np.std(l) == 0: + continue + vals.append(np.corrcoef(pd.Series(p).rank(), pd.Series(l).rank())[0, 1]) + return float(np.mean(vals)) if vals else 0.0 + + +def rankic_feval(preds, dataset): + """LightGBM feval: mean RankIC (higher is better in lgb convention).""" + labels = dataset.get_label() + group = dataset.get_group() + ric = _per_day_spearman(preds, labels, group) + return "rankic", ric, True # (name, value, higher_is_better) + + +class RankICLGBModel(LGBModel): + """LGBModel that early-stops on per-day RankIC via a custom feval.""" + + def __init__(self, rank_eval: bool = True, **kwargs): + super().__init__(**kwargs) + self.rank_eval = rank_eval + + def _prepare_data(self, dataset: DatasetH, reweighter=None) -> List[Tuple[lgb.Dataset, str]]: + ds_l = [] + assert "train" in dataset.segments + for key in ["train", "valid"]: + if key in dataset.segments: + df = dataset.prepare(key, col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + if df.empty: + raise ValueError("Empty data from dataset, please check your dataset config.") + x, y = df["feature"], df["label"] + if y.values.ndim == 2 and y.values.shape[1] == 1: + y = np.squeeze(y.values) + else: + raise ValueError("LightGBM doesn't support multi-label training") + + if reweighter is None: + w = None + elif hasattr(reweighter, "reweight"): + w = reweighter.reweight(df) + else: + raise ValueError("Unsupported reweighter type.") + + # per-day query groups: each trading day is one group + if self.rank_eval and isinstance(df.index, pd.MultiIndex) and "datetime" in df.index.names: + group = df.groupby(level="datetime").size().to_numpy(dtype=np.int32) + else: + group = None + + d = lgb.Dataset(x.values, label=y, weight=w, group=group, free_raw_data=False) + ds_l.append((d, key)) + return ds_l + + def _train_from_datasets( + self, + ds_l: List[Tuple[lgb.Dataset, str]], + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + **kwargs, + ) -> Tuple[lgb.Booster, dict, List[str]]: + """Train a Booster from already-prepared ``lgb.Dataset`` objects. + + Pure training — no ``R.log_metrics`` — so it can be called from worker + threads (qlib's ``R`` recorder is not thread-safe; the caller decides + when/where to log). Returns ``(booster, evals_result, segment_names)``. + """ + if evals_result is None: + evals_result = {} + ds, names = list(zip(*ds_l)) + + callbacks = [ + lgb.early_stopping( + self.early_stopping_rounds if early_stopping_rounds is None else early_stopping_rounds + ), + lgb.log_evaluation(period=verbose_eval), + lgb.record_evaluation(evals_result), + ] + if self.rank_eval: + # early-stopping must be driven ONLY by the RankIC feval, not l2. + # metric='None' suppresses the default l2 metric; first_metric_only + # makes early_stopping track the single remaining (rankic) metric. + self.params["metric"] = "None" + self.params["first_metric_only"] = True + feval = rankic_feval + else: + self.params.pop("metric", None) + self.params.pop("first_metric_only", None) + feval = None + + booster = lgb.train( + self.params, + ds[0], + num_boost_round=self.num_boost_round if num_boost_round is None else num_boost_round, + valid_sets=ds, + valid_names=names, + feval=feval, + callbacks=callbacks, + **kwargs, + ) + return booster, evals_result, list(names) + + def _log_evals(self, evals_result, names: List[str], prefix: str = "") -> None: + """Log recorded evaluation curves to qlib's active recorder.""" + for k in names: + for key, val in evals_result.get(k, {}).items(): + name = f"{prefix}{key}.{k}" + for epoch, m in enumerate(val): + R.log_metrics(**{name.replace("@", "_"): m}, step=epoch) + + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + if evals_result is None: + evals_result = {} + ds_l = self._prepare_data(dataset, reweighter) + self.model, evals_result, names = self._train_from_datasets( + ds_l, + num_boost_round=num_boost_round, + early_stopping_rounds=early_stopping_rounds, + verbose_eval=verbose_eval, + evals_result=evals_result, + **kwargs, + ) + self._log_evals(evals_result, names) diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py new file mode 100644 index 0000000..4afcf90 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py @@ -0,0 +1,3 @@ +from .optimal_stop import OptimalStopControl # noqa: F401 + +__all__ = ["OptimalStopControl"] diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74e5ecbbbb20bb71fd5cd083383de4ce88476712 GIT binary patch literal 234 zcmX@j%ge<81lR4FvSNYsV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##=)E1tpoei8;X~`327Tc_l^pIYrDsDL+l-TRiy?x%gtBBv|wodwhIiPELIM zN`}uMTYeeqCl(awmn0_Z7UpCoff(^%Ml#4Kpg?g+QDRAIdZm7Rd}dx|NqoFsLFFwD ro80`A(wtPgB2J(gAcqtS0ErLGjEszT8FU_S%U|G9Xk;&92MPiJ;Qu}S literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..afaf562aeaa12cebc8529cd916153252e7e3c38a GIT binary patch literal 10428 zcmcIqYit|Wm7d{4;+qs7dP^Ky4~y2zmhCuk{E{szajHmZByVCSS&HI}C`x>JW)xfM z5O52(rBSC*nk=kjvl|6lph{zO6<~L(0tM<|i#9<2XiTSMW?ifoMX?ALyHM&BE&6NE zxx*o8Md@Q7*YcfnUuVue=iKk!IhQ}P+btBN!ilfU|Ez_D4h zbjZP)Lr&Hba9cv4@S$oLCIznC@^%%uEuR<@*{;pn?BIH{#1YMDb3=}ny5#z~7 zqF)fxnM^ztV+I)@qjM2Kj7R&U>6FN)6HvuRL@u_(%%*uJ%BKaPU*Mu*Je`U_V>|{A zVZdTJ3%!?^xWGghZDb~$h)0)>LXEJLoB>wp441+VnY0i`2247YSYj^CaVaJ)3Okuw zROU>E6B9~xtb7Y!)Tv!P9F%wTSky#O1rl*sU%V8W8 zW+O`iY7YE>BVvLJ%*Io3k+aOD`S@}=r4A$}LIKK_d#Zbknc=xebdD2%qmV#0Fg>uM z>NjOITgqcl%QGBklxQd-G6@b`4dscas^tN1nv5^2GXilggX*TIQ)$)Z;630F6+wEq zi;LjHm$-OrPAtE?k6BD5;tOE?InFW##vI|(ne-xbhzN82pyPB@RpK1C2>JnkhnGGR z=iycHl$d5N@{1`{;leB!LSP81NNNe>I2{qV$lE}G`q>1hin zVKyEM4lqx~c|Oex#1O+zonsb-NQ@ZmVj@1X7gbj!U4`#aXqezxj-Q?$w3Nt^IFy)K zKAnU{96m4;i7tqoAP%rxG%mn`nx5{1Hjs|x!qG^AOR*6ggDlia?lLTI@DHpk_tXpo zmGRU(tZmT3%n}hB5~HX&LRplBEaHI4&eBytCJiir>) zwc?yYiAG97J~IKm+(eHAJ+xUX?h+tP!swvo)6`}!qbu1myvpQDK88{8S`0_y#Ln zAjST2B1J?r<|qS8pGhqyXMi2xBe-dFk>>$L%L@un4w;pK701L=pM5hpI>12%nt51< zWc|SEnRx2#IMcI#VBkQo#I1y5!0Z8Zgv|iBX!uA_M%c;t82Fxwf-0{EzzZ2hc!2XV zfc2>O)nCFdhG1?KPZRvNQk1#~|BAjriCQal%^^`Q!6}5* zNwj296c=SBUHScy!AK@E zC`O|FxD_A~CIs1jp+bX>!O}S;jGi};S>juP*>cNgp>{x<=)=%Kn1SqfSEx_-P-e%q zTG`aNP1Tq!MNd<)C0K0iC^mI{Vup%s3+43v+KQBe@-^iQ*UiPIJ@9Z88yR@mi*5UI zhC8O_^0QR!_T~&P)!v7*IF+?X8~<`g}UeTzPx;Gd6msA=bar} zrVc&;_0h6lLZK|M2n7(Bo`>hoG+*}JN()PCQ^@Kh-IY2RS@z=Hdx;fKuAaVbTrm=~ z2pSZVwwEjV7$+(&$k6c7g>WR9UQCIK5e_sFF{tO;;2of|w7~ZOC7^)dUaF>S;SzQr z1}*z{Q&v@n7jXPx$Ut8%&kfrvwt|b1UCdfT-WALli&j_ec+uNX@b<{wp7n;j_dxFK zU8i^D{5Pa6Q;Vw6YWZJGp!0VsM57t}BL{Z%Z;CP9; ztiM2AqG9d5174@PSe#WXf*9e&FmA`N>!etfGP7_pAp6@pdj260i3dwjJOT}PBI2J? z7Z*y7iW?2?b&7dvDZ`z`<8#ocx~n0YW;s~d_zp!6?Do6^}Gzx%X? zay9*$qCY$2R_E5_vs=;GEziSwPwciWwynorzb82UmG%Jj_TFKu{(TD#`2$#jdlNCc z2;NIPA1ZdH-HfgsKmkd1H*B?Yl^9B840#^#EUVALuDYv}?K+k~a0v&=cw-iJ-ud#* zTakJm54t-N?AbLp_3xI(XKQi3YLaS2JU5Uuk2~m&q>bS{QtdqM!#ffS;gzrISi`KY z30ktXQtg)=B7!+dQmux$tYHCgkGF}q1(OtmMWJz@#07%KfgPzbJ_~1wUA2X;-I2=H zF^er)6Km8YnMeHDk$`vmjw&c>BeIq_m9Z$A)47tA7K*gx6UEHgU*WQzLfGkeQIv>KT~7wiSRs|s6p@rIPMs(lyi&F-njkt!JiWf{QZq&+wPm*@fB_`0;G zN|s$@CA+>(YERMspGb547vL#Pu*!b|dTa)d*$uIF+sXcfwO6&1bx0jwwn&iUP9_+o z4zL&O6|lCCgUvgL6+1uY?fBKUBdII@rc*oWXS+UUYe=ooK`;(0m7rwX1Dlp6JWyJ|Fks=!Alo|8slm^mWA ztrp(f#J=SH-edo^{Mt$WD0bPe#oDw0zT&{Ics2eeh+Iaot6%k8hJ0O-V3T4YbtImy zSl})o1s_SND5vN$tYRLCh>=kY4l9N@+)u1{ZjMP0;D&*|9Nx9|}YQYF?xcVn%{&c+6Lu zM_5*|3Mf3e852N|6w6(%irbw+miURH=ue-&pw1gFgh(7t(L-!h)#EG#W7E9S%!Ax) zDRL5?St{9J5yFANZuN`Qq27ydVTk1}CPc+jWhSdeYt^pQIiVB@p%orVmkgpI`EZ7+=2L;7QE^fiPOG%{`D|%;J(JkRC)=>}-5?w$g4>*9W{S0YhoOR_w(ce+<56S+ag8!84KXt2lYjSGKe=6@k zFPmC(mX*hW-;L*9d9L8=m3_S%%O5vo^FQ<~@B|w!V7~_sgkksnt;4 z(Ua2`8(RyFy>er3q4AL1cqrfaP|kk0A+Xk+ZwM9|hUA8!jYEZlqw>Mge8Xco+dYpj zXDWJ`f_G5%4*ncMb0c@1?v-N&&wkmnf5V&iJd}4H+3d_aPZR_FE2dTNw;bC>$~Odo z-_Guw^=?D+tBW~X(OH)>6q^EXwZGn8XbN6Cmm68>D0=(_&!FrX+%RnRZ+Qmup2u?| zRne8hvga^HS9))m@0<@6&WGjmVGOc7M1m~2k!=XDcyrRKxF)_K6voc)Mk9_u9of z-eA$&Sn%$Zy?YDZA=wK(Hl;1^&>ins(bZZClzM;e>b}<&STU?jy=vL+qMG+@_fbAz zvQPHz+c>}F-Iw=1nls<^HLiMJy#QY5azk9z(|Y6NYbSGO&|rtZvAjC<)`izE+;(<< z6Ss5Phi1UA{~ULLzPw(9xN*0$}TydB&1l&9@YVeRl=FU#&=uJ$8mJv!l|8-~2+@Rsc` z*s7tUP~R)p_vY*SbJk*W`&;{7-}kqD*PZ}Z=q%bA-aNP}75v9!|FO-fy#I8*<;*Qs zZh1nsJ#p;|VD*;0z@nvl-B9RxScbpmhktl*^Wu*?ZpHq&FaPL-+&qzULXVCvnEivi z@49^}i9-EBx&GkBNWT7&y!+_pl|M~wtV054||Nh)N=WhECztX(9wIZo?UmZU;J(ZPRG}|#>>87ZnO%x4$3}_ z6>z9W4)u=QC~&B6*1+wI>^pO7^n=m7Z!$NElNQz=+FXG7{KMJrpL_2Q&e_#mP-EPI z4_%$xZ7`8f8Or5f{nC0!_8-~w0&HxVPLTL^&_y<4MGwEPP;b%jOsrm|5sGZ}U&A_b z1|Sp$?s(Eljs%4AX>|~~?*jVw<_tS9$?(iWZNSQDz$14 z#W0gjCxY~N(2ci!_#+`W{=iWvnP%Z9JwV&SVf>Lvsiig?X46rqt{XW!I(&X?DtvZ) z>gfyNr_M|%u5#70=cY)FPpf%y_}TE|6JsOck>LxIN~QYTcvW?I{P@Jwxry=NF~y>d zpBy7scZv;84)A*wxapruv*Z9rE;7{f(kYzrS0PjErR&2eo(#fp@bp8ifQQ>tWVa;0 zKOh;NyM*sS_PeXpCpxRqv<)`&7QOYwmcC;1zGB-D{sG}_FSZ2$bpg@9#Xf0kH~O|s zl+(R!M#=)#-?k!UqdbkB?ziB|~G)>jG zf32g9o%i)XZ4;`-Xl%XjFc|mTx78Tu=-;|$Y2!)NtLpieKytL!>>`*Rti?kF9nKKq z68@x(H;_UGe)1I&_$RSof8eJx>H)O=fx7a~X~R_Km gy0LkS@)aAKFt@v}SU#;u>*$kPO})RNkRWgVZ)^|>Y5)KL literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py new file mode 100644 index 0000000..79aaad9 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py @@ -0,0 +1,217 @@ +"""Optimal-stopping / stochastic-control strategy for cross-sectional signals. + +Entry is a control policy: a symbol opens a position only when its cross-sectional +signal percentile is at or above ``entry_pct`` (i.e. it is one of the top-ranked +names) and the portfolio has fewer than ``topk`` open positions. + +Exit is an optimal-stopping rule: a held position is stopped (closed) when its +signal percentile falls below ``exit_pct`` (the continuation value of holding is +no longer worth the risk), OR after ``max_hold_days`` (time stop / finite +horizon), OR when the position P&L breaches ``sl`` (loss control) and the +position has been held at least ``min_hold_days``. + +Sizing is fixed ``notional`` per position (equal-weight control), unlike the +TopkDropout cash-allocation heuristic. + +Wired into qrun workflows like any ``BaseStrategy`` (see ``PortAnaRecord`` +config). Mirrors the API usage of qlib's ``TopkDropoutStrategy``: ``Order``/ +``OrderDir`` from ``qlib.backtest.decision``, ``trade_calendar`` / +``trade_exchange`` / ``trade_position`` injected by the backtest executor. +""" + +from __future__ import annotations + +from typing import List + +import pandas as pd + +from qlib.backtest import Order +from qlib.backtest.decision import OrderDir, TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import BaseSignalStrategy + +__all__ = ["OptimalStopControl"] + +DEFAULT_NOTIONAL = 20_000.0 +DEFAULT_ENTRY_PCT = 0.80 +DEFAULT_EXIT_PCT = 0.50 +DEFAULT_MAX_HOLD_DAYS = 10 +DEFAULT_MIN_HOLD_DAYS = 2 +DEFAULT_SL = -0.06 + + +class OptimalStopControl(BaseSignalStrategy): + """Optimal-stopping long-only strategy over a cross-sectional signal. + + Parameters + ---------- + topk : max number of concurrent positions. + entry_pct : min cross-sectional score percentile required to OPEN (0..1). + exit_pct : held positions are stopped when score percentile < exit_pct. + max_hold_days : hard time stop (finite-horizon close). + min_hold_days : minimum holding days before stop-loss is evaluated. + notional : $ per position (equal-weight control). + sl : stop-loss threshold as fraction of entry price (<= 0), disabled if 0. + """ + + def __init__( + self, + *, + signal=None, + topk: int = 10, + entry_pct: float = DEFAULT_ENTRY_PCT, + exit_pct: float = DEFAULT_EXIT_PCT, + max_hold_days: int = DEFAULT_MAX_HOLD_DAYS, + min_hold_days: int = DEFAULT_MIN_HOLD_DAYS, + notional: float = DEFAULT_NOTIONAL, + sl: float = DEFAULT_SL, + risk_degree: float = 0.95, + trade_exchange=None, + level_infra=None, + common_infra=None, + **kwargs, + ): + super().__init__( + signal=signal, + trade_exchange=trade_exchange, + level_infra=level_infra, + common_infra=common_infra, + **kwargs, + ) + self.topk = topk + self.entry_pct = entry_pct + self.exit_pct = exit_pct + self.max_hold_days = max_hold_days + self.min_hold_days = min_hold_days + self.notional = notional + self.sl = sl + + # ------------------------------------------------------------------ utils + @staticmethod + def _pct_rank(score: pd.Series) -> pd.Series: + return score.rank(pct=True) + + def _entry_price(self, pos) -> float: + # Position stores avg entry price under key "price" (see Position.position) + price = pos.position.get("price") + if price is None: + price = pos.get_stock_amount("price") + return float(price) + + def _pnl_pct(self, pos, mark: float) -> float: + entry = self._entry_price(pos) + if not entry or entry != entry: + return 0.0 + return mark / entry - 1.0 + + def _is_tradable(self, code, start, end, direction) -> bool: + try: + return self.trade_exchange.is_stock_tradable( + stock_id=code, start_time=start, end_time=end, direction=direction + ) + except TypeError: # some exchanges take no direction kwarg + return self.trade_exchange.is_stock_tradable(stock_id=code, start_time=start, end_time=end) + + # ------------------------------------------------------------ decision + def generate_trade_decision(self, execute_result=None): + trade_step = self.trade_calendar.get_trade_step() + trade_start, trade_end = self.trade_calendar.get_step_time(trade_step) + pred_start, pred_end = self.trade_calendar.get_step_time(trade_step, shift=1) + pred_score = self.signal.get_signal(start_time=pred_start, end_time=pred_end) + if isinstance(pred_score, pd.DataFrame): + pred_score = pred_score.iloc[:, 0] + if pred_score is None or len(pred_score) == 0: + return TradeDecisionWO([], self) + + pct = self._pct_rank(pred_score) + time_per_step = self.trade_calendar.get_freq() + current_temp = __import__("copy").deepcopy(self.trade_position) + + holdings = {} + for code in current_temp.get_stock_list(): + if abs(current_temp.get_stock_amount(code)) > 1e-6: + holdings[code] = current_temp + + # ---- optimal stopping: close held positions ----------------------- + sell_orders: List[Order] = [] + closed_today = set() + kept = {} + for code, pos in holdings.items(): + held = current_temp.get_stock_count(code, bar=time_per_step) + mark = self.trade_exchange.get_deal_price( + stock_id=code, start_time=trade_start, end_time=trade_end, direction=Order.SELL + ) + if mark is None or mark != mark: + continue + rank = pct.get(code, 0.0) + stop_pnl = held >= self.min_hold_days and self.sl < 0 and self._pnl_pct(pos, mark) <= self.sl + if held >= self.max_hold_days or rank < self.exit_pct or stop_pnl: + amt = abs(current_temp.get_stock_amount(code)) + o = Order(stock_id=code, amount=amt, start_time=trade_start, + end_time=trade_end, direction=Order.SELL) + if self.trade_exchange.check_order(o): + sell_orders.append(o) + self.trade_exchange.deal_order(o, position=current_temp) + closed_today.add(code) + else: + kept[code] = mark + + # ---- equal-weight control: target notional per name ----------------- + # candidate opens: top-ranked names whose signal pct >= entry_pct + rank_desc = pred_score.sort_values(ascending=False) + held_codes = set(kept) + opens = [] + for sym in rank_desc.index: + if len(opens) >= self.topk: + break + if sym in held_codes: + continue + if pct.get(sym, 0.0) < self.entry_pct: + continue + if not self._is_tradable(sym, trade_start, trade_end, OrderDir.BUY): + continue + opens.append(sym) + + targets = held_codes | set(opens) + if not targets: + return TradeDecisionWO(sell_orders, self) + + # total value (cash + marked positions) -> per-target notional + total_value = current_temp.get_cash() + for code, mark in kept.items(): + total_value += abs(current_temp.get_stock_amount(code)) * mark + + target_notional = total_value * self.risk_degree / max(1, len(targets)) + + # ---- rebalance kept positions toward target weight ------------------ + buy_orders: List[Order] = [] + for code, mark in kept.items(): + cur = abs(current_temp.get_stock_amount(code)) * mark + diff_notional = target_notional - cur + if abs(diff_notional) / target_notional < 0.02: + continue # skip tiny rebalances + amount_delta = diff_notional / mark + direction = Order.BUY if amount_delta > 0 else Order.SELL + o = Order(stock_id=code, amount=abs(amount_delta), start_time=trade_start, + end_time=trade_end, direction=direction) + if self.trade_exchange.check_order(o): + (buy_orders if direction == Order.BUY else sell_orders).append(o) + self.trade_exchange.deal_order(o, position=current_temp) + + # ---- open new positions at target weight ---------------------------- + for sym in opens: + px = self.trade_exchange.get_deal_price( + stock_id=sym, start_time=trade_start, end_time=trade_end, direction=OrderDir.BUY + ) + if px is None or px != px or px <= 0: + continue + amount = target_notional / px + factor = self.trade_exchange.get_factor( + stock_id=sym, start_time=trade_start, end_time=trade_end + ) + amount = self.trade_exchange.round_amount_by_trade_unit(amount, factor) + o = Order(stock_id=sym, amount=amount, start_time=trade_start, + end_time=trade_end, direction=Order.BUY) + if self.trade_exchange.check_order(o): + buy_orders.append(o) + + return TradeDecisionWO(sell_orders + buy_orders, self) diff --git a/code/tac-qlib/tac_qlib/data/__init__.py b/code/tac-qlib/tac_qlib/data/__init__.py new file mode 100644 index 0000000..92e6e90 --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/__init__.py @@ -0,0 +1,25 @@ +from .config import ( + LakeConfig, + BAR_FIELD_MAP, + FREQ_TO_TIMEFRAME, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, + resolve_lake_root, +) +from .providers import ( + LakeCalendarProvider, + LakeInstrumentProvider, + LakeFeatureProvider, +) + +__all__ = [ + "LakeConfig", + "BAR_FIELD_MAP", + "FREQ_TO_TIMEFRAME", + "UNKNOWN_FIELD_NAMES", + "timeframe_for_freq", + "resolve_lake_root", + "LakeCalendarProvider", + "LakeInstrumentProvider", + "LakeFeatureProvider", +] diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ed1ead6c1314a3f25784d453e54a15a8a04baaa GIT binary patch literal 522 zcmaKozfQw25XS8!O`4QSKoK(=$^d-1wKHo#R93UN!wVU$~jL?%g{Dd`6-nVsfi((XGjszxH zSi}-Gv4uk%Q6L535*M3$i+iF-ioz$p2uOg@5XJT#CZ#JUEIey}6uK&C(+jqnrGsRs z$~*NwG+T}K9(3wGwbbl44&bN@N3Bky*{^pRs&X9dN8OWXUKMHWP%Y<)U;{}7gFz-? zAlXj7V)Q1X`BbtW zG4!Ra=a!B^!N4`}42lN6fmxy~X++_Ia?fU;`@c{5qxm?KKNiXZpqvBTqj&9yzSgUB zG^*zGY-7xmQv7VlT?3}E)H zBR9$qtU59@R)Lp9CQYS)5;cz6>P2l;CDoH&r2g5_JZU#0KQ&7FC;v!D)z*Jr&z;$s zT`+c@`XY5~?mctwx#!+{&bjBDduIQwsK`aYlR5py`}*vBkCM+=I~r2?l8gIc>CKV zZ`Vo(?*!PTm2Tbxa1mb&rI#;(vXn1_(#MxWS)rAcd=;kMIp|(j%pO3*X8& zzG)pPqlgy>zUfVZZ~lnVIWXdzwFKIx{s6DTh~tIs9wx%w5SUOrHWnVALYx$i$CwEr zk`R#CJPcT&PiqjfZ1G zvv?yMiHOY-o(fv=RHLJTE|-geXWwvl58K~;p^xo7KiI4_eRKcaZ|>iQ|4fuaHw5X_ zl_8_!4H)sw{rkCYDI68X5Em6rweW=)m~I|RoQ+kIG%%3{Ce*-8H!v|MXSS?tOyHy`B#6yAq3z1OjKnoY!q=H_ zj1Px6V2;?p%!GM?8H=MRC*>$^-qN-^#?nwpxn>@>thgc#4Et-LGPG7tpWhGXNvzgdw1t;tDlE2;RjfVgqtnWXUOdYugo z=2jc@tpu1z9F&L$Hpck)jF_Wu=!1?VK7%|+oD)UR)hH)Q0&;za4@7`e-CQgdmo&W> zgYZsY;-rayMWuVgAxWkC!=j`*E+(+Ta}m`7+f=C*UbRHSF#w_}-GU+dnD%}ws#;r) zV9){;z*@5fD%IMiw?n1U6Y(hoqbC4E1FfxkODnV-)xo0^Dm^0zHvnx0&@|8*jmM-3 zXc;$Jln@A1Lf6#Nvx9w?+2M=q@cDthvxD6OeJUAO$qAK=sANbbr&Vft zCSX_TctU_>PlU%OR7)fdQ-mUMQBZBu@yJwEQ0eIzE}^=(>2a2e#;0PET6Cs+kUe|8 zufLZa=)M%7RofUB0@YS+p$RTFE~qZ;n6p!{u%vp#DG?^%*|7*W9HyMcH@mF zQ^Y$IPf1RrMRJ>vImVd9sNW&pqYe0t0@^7Od<`)~1T4u*16%@(fUFr`lNBa%3lA7g$;zpiIF(4mkpzlw(q)pq6HLAx zPX#6{U{fui0jd>*DIl~5R;9YGa*-*a4jdR= z{b(aRwM&;}cU_vU+W@M>Cgrd~0|=ko_!j^ci4<|WPQpmqGl@5}<(b4Iwmdv(i1QXu z5Zwg|LK!_E=xvlFuYv~NWYHGl@M@^yr1$B4SZ$_kpwEMX7>`T~jAXc56pu??jBqm% z35UWG!=Z8JH0VyD#~dAH>SJ+cjEh9BbD+GRJ%s031 zZ?r% zI7ajUS}Me*!zdn$3NeX^31IXYF$`Wr0_ur)I3__D0o!3_00(?V7zx;f%<7&;^%U+e zxm@^$*9vx;0qt#KE@4Ux=8L4UY*Q$MS=Yd5rw;fBO3xJ1In_K7j|$D2T5E@|o{xua z2q?hBVX5FAxERA-2Wd=60)#ix1o&`f7R)vHAMl4v6Kt?4;83Z!sM>RzRW0KJn7l!- zk`R_?#|>daf=)o8x?mN+{V7pEsvY76Ea){-b>i7syjT$%GzOi)$fY?1*6c+Wq5Vbp z6NjO=O{^Caj?%Q<|ERoj@x_G~7lRAImD5Uj>zwl|clo2@o$C};ba2B$~<3 zlMo<7NXuIwxTGPhr6Jt3@@2dgY8$-Upti%i9cl*zybh?HP&=V^LG6Ot&6o3Ts67x0 zd!R1jEBPX*i}@uC2VI&jFS!{ z%sFO>H7cl5SB3(#CO{N~iBu~XbFj_r0OJTl6e43pGzFVBF+4HO53ZJQNk*iyj>b5!hEw?V+LZfJI{wxr{ z=5rG2tq=B++nG}EgH#gyB~PVD1IA`7_}HEcU?00oA8~4p%I^vvvXIk>v-lf!F=S8LNuyXvfFHk2mY!N=*JvKg@gl{yCWBf)abe^Q{q?XPr-UF-cf8X|S z`tMF;+Ir=-UZw49y7h(6Z>9s6WbdW4Fd8Y!bi6z|?f0sz3({JFt$5%~0eO!6kbStEs3jz_Y~< z2b>BfOSawz(EyBGBQ;enbYevYqOQY*d)d^JfZ-cuP?$1JGp(|>HSK8q8J6qaawpVQ zizZ;Y!nJyGSo%^6wXd*4#G|&Xv_sw#1IvuQ%@(5vMwaGb12XnC=`&jLwb5tL#2TOM?2bg3oEk7UV)2<6 zzO3q@cC)h678B!Z|?310Q!s0E~?rOi}(2w%2iB08<% zL1JPS;sYEtYFr4KHNTxC%R|Y=bt5h~+DqH2s)=XuGT-l*ic4aEN<;rfQvPbdurX9Vy2Acj97Jzj` zvpNy_kI+C%%JM7?oF|dX`BNHbF=Tp+mdtxv%Dl{@?^1M%-tLx`zac^>R08YX^4x&9 z(z!C_r4lYu933_cr} zf08VIY85UlsXuuYKZ6u`Hokw zgE*OsbJZeT$m_)b*zT9>MG3xVuwfB0|2SU>K;RmrR`gq?ZdW0j2ZrA8zD7{rq}|%I z>x^dZ5e}$S`&bx>5-X{;2$*wL3Zi4s93T-6A-?+qJm6SdEuHO08jrNV|D>z+NM~nb zOKW3WOP~V901M%`2)%|2eE(LReK$jbmVj2By^xMOi@$q7(pHF}^B4(VCkvbg9fnMW z>Kukt1ne!EK*u#AurdAP1LE+zH!;gWUU0_PjcPZXl1p<@5_dfU*;?H}sa8&86SLsZ zupiQQi?4BL8W%H|o&{*BR&DbtjkgZ&?lsjSU5|)ZNN}&sJ^Gz57k05&65=MerL$b` zcoRm7Ux4w3X+E zGuetN_`i5{;p);krDFeT`6tzD)pEtrY}Jm%>4oW~8%ouo)v#RE{=J2)==?g@I-yh@ z{J3p(`lAz>!zbm#CzZpe<*L))Td9iY$?q+6Wf{!iBFd}g(f#)9&YA}Ymk+M&UOqJ6 z^QdOmgZ}0IRp07MYrB-1j!exdx#pBo^W1#jqw3lRCznsIT2{}k6)DxnGu0>L>Jv)! zsrlY)?VbnhGON@!J*0nK{Asarce?say8H~d>U#rs29^RVR~1i#x$C=KzxULC zT~@x>xX`#luJ)`Rn{QOgTIPCiN~nLKf5rFV&i8k&_IyzHuvFgNmf77Y@9tD~pHRwA zX3D$e@@}QPXRhy&uVV4!!pRlOhpzWst8E{6A5t>YmSH+&rc+@~D87>!U$^Y*R(w5k zz1fPLn8QlNfmQmG;z9U`Unf7(1Jzd$7hP1aa?P&b(9~+7C+8+o~m&hv%ue|;0 z+tj)R-XDuN9{Zr;Tp96YS?#$t%a^Srl$)};W@PY>KSOMa{-MAY|6k)$0{tbVIkt~k z2~+|81U@5RYYF~l8%u;eH154c6|BVr4O<=~+BL2F<}>D}um2u$Q<;hamb6QP--p zI3gd-ZxcD>qoxJw?hYD8-J*vGdv9CJ9V0 zwy!vn&VBCFP3-8XaX&Q7(&%WDX20~DnFob|cU$bEW?8j<)h)$Y$m|MuA_$cw)fvOz zTO;A5pt|w*4^~TIi>eJiQo;8b^yfe>VAZW!-j-Q04N)a;ikKUk-O{a=Pm8IIHEL6d z$z26k^=&HAdB(1YZ1 zQuYP1JF1t?EZqbH)BDMVwF`1hyS$@g{>&r)&IiTI#Vcs_!23zX- zbBez^;~$Xy1B(A*#(!1zUse2nHgEl^<+xJS{%OT(_q)^s&$4Ib@_XKg6<`O`CGA;n z>74U_ey|Y!4iO)le{HYn@za~mDdve5!QGpO`rxb>^b{r@b12YbdCv+r zEIOF1$TKgW!9A698cEz!DjCr8MiVf)@Zt^$QY0kM6A4ni@ozCU0|lfON)N0^AHM$n z>*)i>ABI1tf9w9j{n@QAyVKra+7Z-lC};>UAyXTK>T!|ru@`?;y_WZ`(69MCEvauq z0{jRdrlt7BOOkHUW(8#F*L`a{_Hd(yd`hQ;CRhEud3@ zBq<)5;G0YbZ**xS0wpSNCI%4#%c?GxjmG(@2!=f@``Q#2$@SP-7QTA`3x9#<){>Jr zEYR~*V;Is-8os~kG09B~y@iXv!o@NaYkGn~I~wRRZpDR`-+KXoH;J$4;EyzAJ-Xo{ ztwZF7jkI3Y%2BPns+DcHxfK^}8xDuHkK8D)u{Ld-bXm)O^pn<$q%ISgb=R63_L zj$)n#$Y4%#=vTNm&+2mY02(k8#-{KuHek2-CW`RlaJVB)z^5-wLU7TZO@P5dSho?@ zdt}E*HvHj+6ZKeZ4{q_YY{BmZfZ`F{>bnxW5WM({VD9sH5Wcz(slMPvXz1%6zB1Uy z_FU}0G7ucnzLn}91T==Nz-WDgXD}VdNnKp0(JH)x8zjUID8Yr2d~|2uI|CJuf}_||Qu`>8DLoIAR10qASmF}H8s3edWZu)3htj$sF3h0n6iE?duF>0T zF}zEM>oCj^)}lFneK){qqICZB22n~<>xYPi8McmMzZ literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a85628d71d12cfe5b18b1c884c5d829c89594579 GIT binary patch literal 13488 zcmeHNdvF`adB4M(c#;74k|^mUQ6eGpVZE)lDAJTl92KNE znlnV{m?3DOF>Q<*gGTr@MNKht&>XV_Eir4*8nXp$x$?50oz!y#o$zapx?=92JLU;` zV&%bdQpXalh!ay@*xo?Az9%OSUd_vpE+#*6_!yA7J2k4AbK zj+IzuNJxxCI9?E$UN(G==a}9xMjGImQv%EJZSBkuE1VzZB__(A<6G?ZQv-0VB--nwwvkc!JL*zJR&uQ*(i^-LfX*#T3T9~?2))A z3BxfyE-f!$`gm3v7Wiekfu0^16^D}|ti+2rdW?a=nchg8%YCXu*?W50*(lU+1uBk0 zn_RL}?Y09xF9__=5VD|A~~>j?aS;hEfe^hKh4b1zWB z<(F)?-QEl%=D6C3oD_Iqa9m_q$RFZbPVz#87a<>5$&co@#HLN!(lJ8`F(ScSHf>@A zq}$Fw)2CoP1Sv#XHNh-7HF*|#YK*Z`cmU_q)6)q=<(Pj`N(gK}-x3P-^HN0OW1$dK z7Wo*iYghyhBdJl~K6IsR4{;gIJw3lCj>UQt(cR2bjf4O)bfAfWU3GS%rzh7(g5^RQ z`O7+B8SEcdkWBtmBnC^u#)g0pU@V6;bL^PdV*fRsxB#s<*myi4u{eFPJ77{w9g(o4 zn7W`B#ddrM^Vz6kJ{^Zdz^IhB=k~j{85B=$_IidFx23xGwLBQ70(cUNCAi@zABraW z`+4El2n{fz>rmT?Q0LJ{yE;P0+MZDCU2rzr6Y;)Czv4UH{do8BZ*=F1yW5UEdQ$O9 zFsVL)jq#zrgb?Zz`19k{5*rSYLrM1dkOl+c9Q+V?L6Ctzdg&2JT%#ntOr|uL8pJw9 z;icR+M~_e^seqw7U{H*}M8&X~!z4^rF-H>@cwvrKOh}H1Ju_c4w!%Sd1 zTam<8oy3-*F{La7{gIgnwa^q0of2yxx;{UOFcj3cJX68?H2%pvMWl_DQOs->;&h`gfrN#;PqID-0qJN525EmI~10M_`AI@Pfo> zr-;E_MUDy}k1hdVlL3N_G2#%S6HB0&2oFt?v;@p#$rQs7r$|9h7oXu6}NtlLokpitb@!(_!IjexCOjPcVb zj1h=*)(bq#F?~WJ2B5VUMf3hvt*j*{u@QCqKw_a--_3x8kBk76lQzAq0N5JV_cqXp zM8h||)B%a-XgCk>tAOtMM*=LfXI z?N=0Jqc+-vSvC*4ii+V=TR-6kunDOsD6bP1nu<^pR>`N;hL)9DI{(Y*GttKApw9LH z&e&wjAfC~pAlo=gKmD>*+ZhvOa@iolbWz|e1^w-S{_GqjTcwp+wLx6Rq9EG`*A!*J zr#3gOzrnQ+uCo`4t~_F*uJoFyf|*+f*FU(8)-PEV!26SwY!J4~CeDV_nsCUbub|qY z^-xf+{{Hc}vX|{z&2cF1r!mG+0UjLP<7O0=`^KG2qb}0-jb8|$@&`Hz1E*~$AxLn} z$9QQs(G>A?<#8hskB))L2skc+Ruh2_QB_LCJq*in49mnK@nMN?j`AaXG~W*N$)5WF zM!&CV=~x81M<@@Tm#`Uw1+}8XJo8 zsQB~Tqk@nSga8yPzG$J!2I3x6WyK2672+a-V&O+Y-x3wi&=?DCF6e_>bMU8FM8XNh z)E`asDt1K7ki_;zdBviVCgAQ8R^wP^Rt%w15l;v)HX0e{6%&dURW5OSRAK{8;UTO} zB$B0X7%dW|OkV`WOz0C}6H_4A&I=4C=pn@@^+pwXL@@!LNg}cV&`b}iKyxjd>G-;m z__nML4kI>z-U1ls-#8;>{Y}ZTJD!@9|H{QX-VKZ1>Xd0}G$qd-o!xVfqIdSt5LdAD zlF{gM-=mB!_dPpRQF+ZaZCfaB%9J-{%UhD>4?O<6RV%IyOb;wnZOl|{oEghjZB8Cu z^i*B_{mI|I?OF3-dDGp8w@exD)UKJazUaK=OgA6MG6#QN+cwpt)a}dE?aS63 zc(3N*M@DLO^F1@Q;t>6bjjCOf5^s)PAANp2TeE4({DH4#vATAu?B5&}cYQS}@AR%@ z$D+UX#)+F}uAiB$&#r2HZQbiluQt8ic6&=_W>sgp?r_F`IQi(J$Dec(yq`04D~8i2 z=S%_$<}X~Dst#t0a8_uBJi!l9z>4B3j>>XKv4=DbJft{7q4UFRG*@B`g}6i*BpS*j z;W3C5S59n&d9V{hbF|O`xkR%)sERK<@B&7ySb&G%x0k68Org(A2J`06Y<1>JLMW*c zhEhG4)D*qc35jBrqKv|SoLT)PX-k2MVFe|_6f|=-&U6{n45DmU0Zq!3D%@!In;zre z%;$+kN``vYMeVeKj&UI(4G;?o^%GF~z>Li4B?M;By3-YsmT+z&WIC7zg@IlQLZV3J zF#YZ-hin|BMNn8H6euG4IAfHs5xNJRvjKk8#X=oK#TcuqaEMS$PB(50btgQDDYAa3 z>)?ScR6*91=A5pCU~3V31SRAXCspPyrG{Lw-Lp`EhyThozXfy>b&oEyP^{1aVNmyd zI*F9ixhP7~rz+_h#vp1lML{C&FU%Z76e&+MxCI5-v2QVVH~u4io&3-C*=#Y`JXJX z-SSpkJ_Rd``cRcLX!DOVl>pI9O~!iRFT#~U7WvWhBl z(4F``L}>6=PW9c8UO4^op;wN~9eJ(y^}$yM)2rKVI}Ux&v?c4?I@dXK{(J2y`q|EF zUDI99b@GBb-hvbUK6x$ z%QPKaK5n`R8nvaJZHpcs+NKMh`i!SO>)DvLZT$RW2j$=P`M>+R;jc|ZHRDaDqsH_j zV`axG(~oKyi2r^i)3MGJ){i;V(an~BAP^zb9NBymmV#qv(vHxC6*dw8N^y{NH5?Rr zn`n{E0#R`zpyqN0@O>L%#)&eT0wunL+d?+IoFBn1IketO!HjGGj5b40cFur@M5oN~ z73~n))pq7(?Tlc)IAj}`Bi{y4aQ0IZE|{YZGa4-bdL@p#?38sA1$sC`QD=aQ6YdF5 zfm!fXl)1GzXplBhF6kzeZ2XFOdG2dnA(zV)=fFM%2zyL+l{BxQMxX8+qNyuSn<&`@ z_5V+RDhfVFV_X1J#mlRRE)a1Mi@*9o#?& zzXBTeiUT~_=!qYS4Dn!3SkzBFTN;CGuWDZKSl^xZaH0=U7{E-MX369&0974uswfm9 zYK&rxv7-=0;)(^Y2jZM!1dpszrgtd}VsBQ+WF_!uI#BNkfj=JnumWr*Do{nBPjMi^ zs_tD;u@fJ42rUQ2hCXsksb+;0tPq4?l>zOf^BgV`!d0Wfm>336iJFfP3>ueW?u9Fg zxMD;i4bg;R7>EcFZ12kPOQlMoI~7|lM-q{%sMy>eRI{WUX({x(KM>2c9bUPqh0*UuE*>rxvQL;jX*lYIHI> zV@gJ|?hQ%foyuzBlH2s&x*hYz`L&*Oy?1o3OmAj@4m|63}mKXQl+B>nckV2_x+ph zl~HRB(;wTZ^2){4>!*xUoqy`MS3y;;zUQZ^*Ii3YCuU;Vs%`I8?S8|Mbo^pfWAfOC z&gNfLuTGtxX`Xjw*6q($A6Te*Bk3;5j~un@aO=!TEHHABSmc&%>h;9Uh8JWWJr zKYKP{B6Y{h$#q3RiDFT2g~rReU>gt{DPZdE7QE1XXTT;L$6=np=xZ3Az^D-;Ja@t$ zVRRCsQxFx}Kwrm9j80<#LKU%r!uM=cedA))ipA#5_sZ<%9d~`qJv01*j$GltXT_9_ z@;>w_mMzs-%?-ctZZP{)dkMP>!=D~?kJymPhfS&Qf%z!@dLCKbNOov=XZb@9v`L zk99YX3T11%gB7A-JUDo5P9N5O`oXMq-=smbOp5r^L%JJ08`}2p{Db0Z;%e zuH7UHv+<&=J^?V*T0ao!A84(I`<2%Ea5N$Et@R`5i^1OuEZ%F8(7Yc<)d|VDYnJ!N z#uCGEyfAIt$Bb+mxd69nFbX@;A6ixd4k!k?EA}yKeJ}45ZY|-~4@e^yo~>|8r%|NR zRTx93;K6S0X1ke2;@^UnJQpbFkd2Q?IjFO=wpY`%O@=R-aZGznVMJ?` z3Tl=xi?tq0&pVuYNzHRv)&Z(Y zCtlaiseTEt(sT_%AbNoj+>-)2gGA&N+4ic?cEVLLI683*u5X+vx#|Tq{t~<3Ai6{? zdTLX<(kN;d9quU(C0E9=VX@q=dI{HN%Gb^WW`o)CN0M!eyLSJu=FOV7R)CN^aO=SQ z=G*JH|J3`lx}Vg&yE@r%_4wrRRC~tBpq+DZ@?uKNc-E(F>;HetP`}HTc?On6-7+o> zoQn31@F!U1Dn?Tf74L;|buVZD0sEo;!TTX!>#wmZJhAufiG`{_2L3w&`mWfRDF-+4 z{HARAKG+re_W$t7H=lef_=51_#akEW&);s?_0tnSJM)t>?>?RESa7b%IM>V=GtPS0 z7Ym+s8PB?ztr<^4+SZ`%j6x_t2qR!S3W>t|xKe;2gP&-bS!l@L$>neAgevd|6&lab zGlo*$GQ8gb*>30-9)=p=9wf)qAtlH?1X0;>K*hjG0$NFMVQt|0R5jip6?r+#K%*=FrKO->-65&9K1q#%MBxnQK+6<@obiEFJpVEGb!WFWDW#jQAOd62g|5$#_;8b`d$pL!&>p2kOv}6V|OABuh#m=EhRpLgWEn$^NNB#9BAnV zE1PJW!Vy>&#Q@4?neZ$m1>7G3I43BmA)y<^}LyyIOnW6pRtCfmVen+jh$H+?SSS-V&b zCf@YuwTsgiUl`6-H!oE0$W-r`mtG%#b^Pu1Z%f&oUD@hm>GETXzUoCs`P7m39BA_C z?Ps{z@GIx$&Sf_4TC7=_YQJuq**YWL+LftkTBzQZsowV5>1_3$h3bw>b;sMMejI!! zn1149`t&!mj|Q{VPo>MBQv1E_Z^$?rAbl4B8J&?c!TH*(2Q)k!wF3>^jDvv+Df9H6 z1>c5@Z^H|G*4MP)+n(`lpXboi`nDhF-O-ix9ZNfp0kYqZn`(b%EN!b%nNcWoaJvzV z{wcxe0za+p(E|oiE0#Sbll2D}fX7rxmgQ?$3m9hA&~w6;*L1l8pJtw=x=bjObVP=S z9P~J7(0MOrUU$~(+vPIZey#)9Tc5uu8Z3u-76l1?2P6#QMQc&WFF9Nq;cAf}Td{Yb z0=`xNYU`sqfDZ_LMWKM7`qj~wU6cU2f_&3Glup@sX@B1PTj~Ky0e5xZWx>qwJwZ_@ zC}oO@;1lMJ2L!@|E8Z$QOFcu$dr1d3sC5hDuk1gk1x&cUe0}7?@{NTfsOkJlXsZC8 z|D4hW2h!Y4o_UZ{3BFk~N4&9+AP<5AesXuD`ex^#L&XTsz=Z@!!Sg|32vVw=Wl@ER zfX*AmO0a(G7V^wbco8f0V7{quI2u)qaW)r^CP^V|g z_{@)HUCB4$i|ZMyP?F?IBLch=RAG%Pw?Q-<&CL^tWK>0My1 z?Y_19wy!1aZ26UERVoZ_OkuX|)<}AF%e*P;*$%+(_Fj!m#%{FF*q=YXP`5c#w;7&k z%%6VaD8RhCi&%%rJ=ovQ>76&WrljY0z0mt&->tsa4!!=^tB>9G9hmI6>#Mr9XL?Uc z%KGXTe9akOvs#$;HD`SXk{$Rqb8K=f*;{Alx6BJ~w7oi#-LU_*=Rn$afS{)E zI50;z1`&8RQIxCur(I9|F{Hk5!h3byT!xB&hQ)u25dnO}8G%%^K}B^N*b92YZBX3r z@`mF+-FK=#jYqHP@>*YbiI)^=oYHijvjNwGouovuL0v_0!XAXT?lE2(NN}nmLpC`c z5#cS2O1e>#Sc1`W@F(tp2;8V28LVd8XI_(e?~>bWuE(hHo}03GKQQ!aAdLUcqtimDcf!bh>FkLJlcGZj7RvBK9e!nw{->b&u4U9s&|U)t{% z-i1#DOi_a9o{^^Mj|@)Q^l2SMum2_G`d6wYOSSxx^8SMI{Rj0#hI-;dYU6KAX48Be${!F2 literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/config.py b/code/tac-qlib/tac_qlib/data/config.py new file mode 100644 index 0000000..686d36f --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/config.py @@ -0,0 +1,175 @@ +"""TradeAC lake configuration helpers. + +The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-lake``): + + $TAC_LAKE_DIR/ + ├── market=US/ + │ └── timeframe=1d/ + │ └── symbol=AAPL.parquet # OHLCV bars: t, date, o, h, l, c, v, n, vw + ├── features/ # ta-lib indicators, wide format + │ └── market=US/ + │ └── timeframe=1d/ + │ └── symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + ├── calendar.parquet # trading days per market + ├── coverage.parquet # per (market,timeframe,symbol) loaded windows + └── symbols.parquet # asset master +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict, List, Optional + +import pandas as pd + +#: qlib freq string (Freq.__str__) -> lake timeframe partition name +FREQ_TO_TIMEFRAME: Dict[str, str] = { + "day": "1d", + "1d": "1d", + "min": "1m", + "1min": "1m", + "5min": "5m", + "10min": "10m", + "15min": "15m", + "30min": "30m", + "hour": "1h", + "1hour": "1h", + "2hour": "2h", + "4hour": "4h", + "week": "1w", + "1week": "1w", + "month": "1M", + "1month": "1M", +} + +#: bar-field map: qlib field name (without the leading ``$``) -> lake bar column +BAR_FIELD_MAP: Dict[str, str] = { + "open": "o", + "high": "h", + "low": "l", + "close": "c", + "volume": "v", + "vwap": "vw", + "avg_amount": "vw", # amount / volume +} + +#: fields that qlib core/backtest queries but the lake does not store -> all-NaN +UNKNOWN_FIELD_NAMES = ("factor", "change", "trade_unit", "suspend_flag") + +#: columns in the parquet files that are not features +NON_FEATURE_COLUMNS = ("t", "date", "market", "timeframe", "symbol") + + +def timeframe_for_freq(freq: str) -> str: + """Map a qlib frequency (e.g. ``day``, ``1min``) to a lake timeframe (e.g. ``1d``).""" + f = str(freq).lower() + if f not in FREQ_TO_TIMEFRAME: + raise ValueError( + f"unsupported qlib freq {freq!r}; supported freqs: {sorted(set(FREQ_TO_TIMEFRAME))}" + ) + return FREQ_TO_TIMEFRAME[f] + + +def resolve_lake_root(lake_root: Optional[str] = None) -> Path: + """Resolve the lake root: explicit arg > ``TAC_LAKE_DIR`` (no fallback). + + ``TAC_LAKE_DIR`` is **mandatory** — there is deliberately no default + A missing/empty value raises so a + misconfigured environment never silently points at a wrong directory. + """ + if lake_root is None: + lake_root = os.environ.get("TAC_LAKE_DIR") + if not lake_root: + raise RuntimeError( + "TAC_LAKE_DIR is not set. Point it at the TradeAC lake root, e.g. " + "export TAC_LAKE_DIR=/home/data/lake (docker) or set an absolute " + "path in your local .env." + ) + return Path(str(lake_root)).expanduser().resolve() + + +class LakeConfig: + """Path helpers + cached readers for a (lake_root, market) combination.""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + self.lake_root: Path = resolve_lake_root(lake_root) + self.market: str = (market or "US").upper() + + # ---- paths -------------------------------------------------------------- + def bar_dir(self, timeframe: str) -> Path: + return self.lake_root / f"market={self.market}" / f"timeframe={timeframe}" + + def bar_path(self, timeframe: str, symbol: str) -> Path: + return self.bar_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def features_dir(self, timeframe: str) -> Path: + return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}" + + def features_path(self, timeframe: str, symbol: str) -> Path: + return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def calendar_path(self) -> Path: + return self.lake_root / "calendar.parquet" + + def symbols_path(self) -> Path: + return self.lake_root / "symbols.parquet" + + def coverage_path(self) -> Path: + return self.lake_root / "coverage.parquet" + + # ---- metadata readers ---------------------------------------------------- + def load_symbols(self) -> List[str]: + """All symbols known to the lake (from ``symbols.parquet``).""" + p = self.symbols_path() + if not p.exists(): + return [] + df = pd.read_parquet(p) + if "symbol" not in df.columns: + return [] + return sorted(df["symbol"].astype(str).str.upper().tolist()) + + def symbol_spans(self, symbol: str, timeframe: str) -> List[tuple]: + """Listing span(s) ``[(start_iso, end_iso)]`` for a symbol from coverage.parquet.""" + p = self.coverage_path() + if p.exists(): + try: + df = pd.read_parquet(p) + except Exception: # pragma: no cover - defensive + df = pd.DataFrame() + if len(df): + df = df[ + (df.get("market") == self.market) + & (df.get("timeframe") == timeframe) + & (df.get("symbol") == str(symbol).upper()) + ] + if len(df): + row = df.iloc[0] + first = pd.Timestamp(row["first_t"]).date() + last = pd.Timestamp(row["last_t"]).date() + return [(first.isoformat(), last.isoformat())] + # fallback: derive from the bar file itself + p = self.bar_path(timeframe, symbol) + if p.exists(): + import pyarrow.parquet as pq + + tbl = pq.read_table(p, columns=["t"]) + first = pd.Timestamp(tbl.column("t")[0].as_py()).date() + last = pd.Timestamp(tbl.column("t")[-1].as_py()).date() + return [(first.isoformat(), last.isoformat())] + return [("1970-01-01", "2099-12-31")] + + def load_calendar_dates(self) -> List[pd.Timestamp]: + """Trading days (midnight timestamps) for the market, from ``calendar.parquet``.""" + p = self.calendar_path() + if p.exists(): + df = pd.read_parquet(p) + if "date" in df.columns: + if "market" in df.columns: + df = df[df["market"] == self.market] + dates = pd.to_datetime(df["date"]).dt.normalize().sort_values().unique() + return [pd.Timestamp(x) for x in dates] + return [] + + def __repr__(self) -> str: # pragma: no cover + return f"LakeConfig(lake_root={self.lake_root}, market={self.market})" diff --git a/code/tac-qlib/tac_qlib/data/providers.py b/code/tac-qlib/tac_qlib/data/providers.py new file mode 100644 index 0000000..d9f839b --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/providers.py @@ -0,0 +1,231 @@ +"""qlib data providers backed by the TradeAC parquet lake. + +These providers plug into the standard qlib mechanism: ``qlib.init(calendar_provider=..., +instrument_provider=..., feature_provider=...)`` instantiates them and binds them to the +``Cal`` / ``Inst`` / ``FeatureD`` wrappers (see ``qlib.data.data.register_all_wrappers``). +The rest of qlib (``LocalDatasetProvider`` expression engine, backtest ``Exchange``) keeps +working unchanged because the interface contract is identical to the file-based providers: + +- ``feature()`` returns a ``pd.Series`` indexed by the **calendar position** range + ``[start_index, end_index]`` (matching ``FileFeatureStorage.__getitem__`` semantics). +- ``list_instruments()`` returns ``{symbol: [(start, end), ...]}``. +- ``load_calendar()`` returns a list of ``pd.Timestamp`` trading days. +""" + +from __future__ import annotations + +import bisect +from typing import Dict, List, Optional, Union + +import numpy as np +import pandas as pd + +from qlib.data.data import CalendarProvider, FeatureProvider, InstrumentProvider +from qlib.log import get_module_logger + +from .config import ( + BAR_FIELD_MAP, + LakeConfig, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, +) + +logger = get_module_logger("tac_qlib.data.providers") + + +def _day_freq(freq: str) -> bool: + return str(freq).lower() in ("day", "1d") + + +def _calendar_keys(cal: List[pd.Timestamp], freq: str) -> pd.Index: + """Convert calendar timestamps into the same key space as the lake parquet.""" + if _day_freq(freq): + return pd.Index([pd.Timestamp(x).date() for x in cal]) + return pd.Index([pd.Timestamp(x) for x in cal]) + + +class LakeCalendarProvider(CalendarProvider): + """Trading calendar read from ``/calendar.parquet`` (fallback: derived from bars).""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + + def load_calendar(self, freq, future): + timeframe = timeframe_for_freq(freq) + if not _day_freq(freq): + raise NotImplementedError( + f"freq={freq!r} (timeframe={timeframe}) is not supported yet: the lake calendar " + f"only covers daily sessions; add a minute-level calendar to `calendar.parquet`" + ) + + dates = self.cfg.load_calendar_dates() + if not dates: + # Fallback: derive the trading-day set from the persisted bar files. + bar_dir = self.cfg.bar_dir(timeframe) + if bar_dir.exists(): + import pyarrow.parquet as pq + + cal: Dict[pd.Timestamp, None] = {} + for p in sorted(bar_dir.glob("symbol=*.parquet")): + tbl = pq.read_table(p, columns=["t"]) + for v in tbl.column("t"): + cal[pd.Timestamp(v.as_py()).normalize()] = None + dates = sorted(cal.keys()) + if not dates: + return [] + + if future: + # append the next calendar day so that "today" is a valid trade date + last = dates[-1] + dates = dates + [pd.Timestamp(last) + pd.Timedelta(days=1)] + return dates + + +class LakeInstrumentProvider(InstrumentProvider): + """Instruments from ``/symbols.parquet`` with listing spans from ``coverage.parquet``.""" + + def __init__( + self, + lake_root: Optional[str] = None, + market: str = "US", + markets: Optional[Dict[str, list]] = None, + ): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + #: optional named pools, e.g. ``{"sp500": ["AAPL", "MSFT"], "etf": ["SPY"]}``. + #: ``all`` / any unregistered name resolves to every symbol in the lake. + self.markets: Dict[str, list] = markets or {} + + def _resolve_symbols(self, market: Union[str, list]) -> List[str]: + if isinstance(market, (list, tuple, pd.Index, np.ndarray)): + return [str(s).upper() for s in market] + if isinstance(market, str) and "," in market: + return [s.strip().upper() for s in market.split(",") if s.strip()] + if market in self.markets: + return [str(s).upper() for s in self.markets[market]] + return self.cfg.load_symbols() + + def list_instruments(self, instruments, start_time=None, end_time=None, freq="day", as_list=False): + market = instruments["market"] + timeframe = timeframe_for_freq(freq) + + symbols = self._resolve_symbols(market) + if not symbols: + if as_list: + return [] + return {} + + # clip listing spans to the queried window (mirror of LocalInstrumentProvider) + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + start_time = pd.Timestamp(start_time or cal[0]) + end_time = pd.Timestamp(end_time or cal[-1]) + + out: Dict[str, list] = {} + for symbol in symbols: + spans = [] + for begin, end in self.cfg.symbol_spans(symbol, timeframe): + lo = max(start_time, pd.Timestamp(begin)) + hi = min(end_time, pd.Timestamp(end)) + if lo <= hi: + spans.append((lo, hi)) + if spans: + out[symbol] = spans + + filter_pipe = instruments.get("filter_pipe") or [] + for filter_config in filter_pipe: + from qlib.data import filter as F # pylint: disable=C0415 + + filter_t = getattr(F, filter_config["filter_type"]).from_config(filter_config) + out = filter_t(out, start_time, end_time, freq) + + if as_list: + return list(out) + return out + + +class LakeFeatureProvider(FeatureProvider): + """Feature data from the lake parquet (OHLCV bars + pre-computed ta-lib features). + + Field routing: + - ``$open/$high/$low/$close/$volume/$vwap`` -> bar parquet columns + - ``$amount`` (= v*vw), ``$avg_amount`` (= vw) -> derived from bar parquet + - ``$factor/$change/...`` -> all-NaN (not stored) + - anything else -> a ta-lib column in the features parquet + """ + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + self._bar_cache: Dict[tuple, pd.DataFrame] = {} + self._feature_cache: Dict[tuple, pd.DataFrame] = {} + + # ------------------------------------------------------------------ caches + def _load_bar_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._bar_cache: + p = self.cfg.bar_path(timeframe, instrument) + self._bar_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + return self._bar_cache[key] + + def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._feature_cache: + p = self.cfg.features_path(timeframe, instrument) + self._feature_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + return self._feature_cache[key] + + @staticmethod + def _keys(df: pd.DataFrame, freq: str) -> pd.Index: + ts = pd.to_datetime(df["t"]) + return ts.dt.date if _day_freq(freq) else ts + + # ------------------------------------------------------------------ fields + def _extract(self, instrument: str, field: str, timeframe: str, freq: str) -> Optional[pd.Series]: + """Return the field as a Series keyed by date/timestamp (None if not present in the lake).""" + bar = self._load_bar_df(instrument, timeframe) + + if field in BAR_FIELD_MAP: + col = BAR_FIELD_MAP[field] + if col in bar.columns: + return bar[col].astype(float).set_axis(self._keys(bar, freq)) + return None + if field == "amount": + if "v" in bar.columns and "vw" in bar.columns: + return (bar["v"] * bar["vw"]).astype(float).set_axis(self._keys(bar, freq)) + return None + if field in UNKNOWN_FIELD_NAMES: + return None + + feat = self._load_feature_df(instrument, timeframe) + if field in feat.columns: + return feat[field].astype(float).set_axis(self._keys(feat, freq)) + return None + + # ------------------------------------------------------------------ api + def _get_calendar(self, freq: str) -> List[pd.Timestamp]: + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + return list(cal) + + def feature(self, instrument, field, start_index, end_index, freq): + field = str(field)[1:] + timeframe = timeframe_for_freq(freq) + + cal = self._get_calendar(freq) + n = len(cal) + lo = max(0, int(start_index)) + hi = min(n - 1, int(end_index)) + if lo > hi: + return pd.Series(dtype=np.float32) + + keys = _calendar_keys(cal[lo : hi + 1], freq) + ser = self._extract(str(instrument).upper(), field, timeframe, freq) + if ser is None: + vals = np.full(len(keys), np.nan, dtype=np.float64) + else: + vals = ser.reindex(keys).to_numpy(dtype=np.float64) + return pd.Series(vals, index=pd.RangeIndex(lo, hi + 1))