From ceb1e196e24761a87d1afe1bde00079b79e02693 Mon Sep 17 00:00:00 2001 From: zhaoli Date: Thu, 20 Aug 2026 23:06:35 +0000 Subject: [PATCH] start experiment 57 (exp/57-signal-quality-gate-gate-trades-based-on) --- .../regime_gate_bt.cpython-312.pyc | Bin 0 -> 17916 bytes code/MANIFEST.txt | 30 +++ code/tac-qlib/tac_qlib/contrib/__init__.py | 11 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 374 bytes .../tac_qlib/contrib/data/__init__.py | 3 + .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 216 bytes .../data/__pycache__/handler.cpython-312.pyc | Bin 0 -> 11073 bytes .../tac-qlib/tac_qlib/contrib/data/handler.py | 254 ++++++++++++++++++ .../tac_qlib/contrib/model/__init__.py | 4 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 319 bytes .../__pycache__/rank_ensemble.cpython-312.pyc | Bin 0 -> 9461 bytes .../__pycache__/rank_gbdt.cpython-312.pyc | Bin 0 -> 12288 bytes .../tac_qlib/contrib/model/rank_ensemble.py | 189 +++++++++++++ .../tac_qlib/contrib/model/rank_gbdt.py | 238 ++++++++++++++++ .../tac_qlib/contrib/strategy/__init__.py | 11 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 234 bytes .../__pycache__/optimal_stop.cpython-312.pyc | Bin 0 -> 10428 bytes .../tac_qlib/contrib/strategy/ic_gate.py | 117 ++++++++ .../tac_qlib/contrib/strategy/optimal_stop.py | 217 +++++++++++++++ .../tac_qlib/contrib/strategy/regime_gate.py | 215 +++++++++++++++ .../contrib/strategy/weekly_rebalance.py | 202 ++++++++++++++ code/tac-qlib/tac_qlib/data/__init__.py | 25 ++ .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 522 bytes .../data/__pycache__/config.cpython-312.pyc | Bin 0 -> 11464 bytes .../__pycache__/providers.cpython-312.pyc | Bin 0 -> 13340 bytes code/tac-qlib/tac_qlib/data/config.py | 202 ++++++++++++++ code/tac-qlib/tac_qlib/data/providers.py | 230 ++++++++++++++++ 27 files changed, 1948 insertions(+) create mode 100644 book/scripts/__pycache__/regime_gate_bt.cpython-312.pyc create mode 100644 code/MANIFEST.txt create mode 100644 code/tac-qlib/tac_qlib/contrib/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/handler.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/ic_gate.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/regime_gate.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/weekly_rebalance.py create mode 100644 code/tac-qlib/tac_qlib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/config.py create mode 100644 code/tac-qlib/tac_qlib/data/providers.py diff --git a/book/scripts/__pycache__/regime_gate_bt.cpython-312.pyc b/book/scripts/__pycache__/regime_gate_bt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..432501c8a61132c95f143a8573feab46c61cf8d7 GIT binary patch literal 17916 zcmch9X>c3YnP4~W`vytyY>^ZtP&~v-A|+8YB}$?sN|r?VkSrSuvYRIeP~D(p!i@x( zaVDUx9f7ecg2wh3%4XNlT03U4`N7m?CrTwdW7lMB&<1A-w^RXHt35JW0^i-z6_v;8_DtD?DrAX`{E(buTI|+UX{mf~P}EXbE}; ziog7i-brtJQF+lxyXkspQ${zuNL(yuD(J@9odm&ZV6^3LXNN_Q#G-sr?rvV-V4&xi z$>}+!b1l4BVY%smzx_(U*FH5jNB!&u6`W!jZYtoTF#tC;!3O3ipLdo4$Sl{Q3r;b9ie2z? zl$W~fn;xUi2j*vo*uZ>XAy`0ubvigjG2ZbhjEwTm&-WrrM!5R8texc7^d@X&;;R zx6dyIrviRzED)G&=f>IT`5@PxrIiQK$}<*hn_tv@g66?Zih8f#9|(GbKsfGn$lN{^ zm}A;uP~LV#CQHqI7#tXM5j7UoYeB8Sm-Y?d{&(H_^LytY?pR&p6#X!7yFjUF}{rI6dJV2UHT- z+zL_Y^Nuk-QOO0NSd=q+SOPb@xH=GJCuw5M$`w7-Dx3^zPO#Lf_lL&^==l-oC!x?(V+b6TM?&-rlj! z&d!Ow-tqCSzAuB`g`sz0=u#S>cNPKW-?%dP0*Soj3 zx3hB()B9!6J1}&Z4>TpY^wz%bY42us@7=wdroCgG-fpI2ckdV=n_$M7yiZagd7sa=@*0%RkQTeqyo1QRS2Q#M?PYY!glL^!yL3r-*R)6Dbx@ZlANJ&lgaX+IZa7tsEu{_CH8 zo8ra;EaPs2-G&mUS=39nN|ephqMl{Ev}b;LeAdT^x@iuz^upOBYKP$5ILdnG7*RbQ z@GZ>wIZ-j~3yiyEuv5LkDN#mGa0n7becoRBRQb{q3oR6c7B)QF03f>I$DM>ILVTpv zU$0%NjVsdHn#iFIwKZ1%>evn6vTse@a8GZGjop}8o)Pp^#_o(wF7HnnYBOf*^{Y!) zzkNBrEf!n~r!>_Wy)kn9K7b=0S^#J|@B(-hmck8Unh29TF_T{gsD3ukWPb8xyle*A zYz?&RjyzvS9z;tEhauwGI)Vr*csZ|_y#y&Gt>BfVmI@90L>0lK#h~I9wE8v08?plc znR+Qz zc=A@M_hB1t<87sSb%kWfYs0ktOG?opW25cp`0y(rtB>?>DoIbSaARsXgk$R)-NdzqN&EMSsS zcgm!LKdi(%8X(dY(vFUi*SewK{1Eu! zC<}T+>Iv`Mv~RILq~;dq#sa?nkfv?k%U)h!f}#b`0E_OK_hcE!4v`Fs3dBnT0Od5g zokWEnK;9-xZhk!I8K3g{CmB)B1!>VdbbR!T=lt-wBcsEQj|`2n zXgi5Y|H9n-qA2$TCgB>Jnw|)XYBu2WP5UQB#T?`Hi`pk2AMqR-JUuijs?q%f5`d^U zgDxuV$l%E%o^y{seqJ;UpE^|xyB5J#F+i~($D(sVG-NNt+}N~7o*!rC5koENg8>g3 z6rzx*8pU%Vs;+o_3k)Z!Xpl4fUg#g=XKMfs9;v%R)ZmHo&=aB(3pb*YWrCfZaHnz} zG(n;q#2S&D6lMN-Q4OO5A_Dc;D^N!TbzrJszJRR~l~<;wC#RskcT&_K3bKJfP}IO8 z@yrJ};Gtn=L{)a4MA^i=s6Z1eDuQr(!yIxV$pK-YCh81AkEq6>6B9`vr2>l%C(#18 zbWTpgG6%^6=8I)8|nl@T|$;NG(?VMO!n*3OVjbQ(do3Q zE^_Rb=Gv${W3b1#rPEO|Q&t%zud9|+Q7$Hn>N5?EQDsaS8;>`{E~HF#?`x<{HDRf{ zZy=QA8IvPAc70-LBKCAb^OotBDS0_fbqLOmw5c<4ETgwwA6OcQcc%5V>-u^@U!T@D zCUt_oD>9TZTCZPNx)2|XUPv2jBS(K(yUXXnb*Rc@8J+neec5u772S*F}| zV{v)$)fY03>P?y4TC-7A^X9qf&%bl&_9emI2X!)yJCfb+9JqZz zXxtkM#t(mcarOM&rXO93DK?ztH=bL5E-{pLx?{2pN7;?z%g5t%s-Z`y-LqcXFVyzm zJ@@YTn&VuI%s8AkhL?w5IT0f_>}9dbuk48retTc0aeHhyKDvDB1B>(1ZA8_MO*!Fc z{IroUI5x^E5I2_SO_#Mqk7deg&|8v>`O;<0@NOxOcP*Qi20yacZfKXa@y5i|TmD=A zHA_!)FjL=0uysSD zNd757ecptqfoWhaqJddz$yO9tJlZCehOAs}| zfmXjr@P?Tx>Amoj#ymtwseF4YZ{e*WkkrDqNnoRBP1pfHC#?;a!FxIUDu7NdUSDd& zKs#Vk!j({3g1zJ`QP+4^FwZpdm9$P;g{+0IoXInbU+k-&HBg#v>cxakFJAv z-GmIzKW(_4FZ+g?hjtKT>A3|T$WITMWw12{2}Y8srG?7M8X(t!MiWF%kbpr-^>bh>K{84! z^|&8fZdwELVD6#4i`+g68-;>k?V-3}V0;!VNX+F0udnqg0}@lv-2yPiygsDXq^>f| ztZ#7(C@iDVIbgo&%L~&;(4&F@YTV0Bh2d~xQOU4ufDN@Eg(o$)05-5G?-hnBsBgjM1t43B6t`|Hw1>+4 z0j%#u>Rm4sQ&&Nt4qW}@|3WzbkaiOtJaC|r_mOJnBxp=-0PFfJu&JQsBth?_qKWU2 zH%PdO^a^5;{C9HDH^^9Yv7Nq6AW^?_xqpZ35c;|E(2dobh%bB{-|;Caxr zVPUZC0BD50KJH*!zUu1$kwa>_|3>^SLZmF+LUmj0@(;GXYK6rA;xth5Cu%GC68t71 zO+;R78$2y){Q-8)>zfWSM?r_7^>X825`sm5J1FX>IfMqXyex~9#Se`fvEp_#axhA6 z5dhvsN)ZS&ISXa13aQrNPyV_bGxj+xRkltxmJa)w{B|ysN@ZXYKtcx&AReG0Z3(mj zAK|6Tuv#B)NoC>CEMj2tS6<;dLA67GhgKmrR2o%SR@5r5v!+$}LaQ$7BUwsYERBX% zBMq+v6AlMiSkB97SxyHh&C_zKQ9##io zCbNN;(HiNcBRNbSrR{h$?Q%%g!^R5=>P? zL+k`p`{e(ix@b4Bh-dr+v74VLAad*26DW5NijG4hDks2r5zMNkzGKT@}i#ms2l!Ho#Wl-V|kD-Wxmn(teP==ogn3Uw$E~$e1j#Zcq_*3&y&rY{TLN>(0dT#4A&;n4`*#+J?lj z`;U9>mc46D??1WrpO43pz`vCMwYBoz>9MU{&M+-sjqRx=qsZ<<^aYo*oanYuRdqv=c>G zWvDBh&~t*$ z7VfbeofUIlZdTN1ucfR-?@v(+70?W68nyX(5J{T#pd#h0KXBDAs{NiIcq{l2STs+f z#_!3_59l|+%>zzan0@)uU#fEE^$e@e34jq6QBGrW7=~3v3pv7WERH zDX7?!)y|8Wv0RG%6HEpSVop?}_Oxe?b3?AQ%fIrOJqF53Z+`Z#0Tf#LTnG-T2=Ra- z3}rDI*+;)u|5od*);0ZZ(8r}*$5Q6ulxFyz)p?_GxpLjwB3N5sV_DlHCo+bz_;#eT zJ+aatH1-JkJ)ktTmBoU~&5@HEYF%`HT3sIRcwb$OA$6q8oheP{N4AQ1)w-)$a5b;H z_6x54tAlCR!FAUW!F43A4enH5fng=82mp670>>YvsFEgHdg4 z+e-$}!mtK| z-C~OjXY93+k&LY-auTpcj^DF6AO+nz0p)flq)m?NzH2_{)QK~H-~3*4>WQb`Ym4f? zH<=t;*|pYmFm~w1$nwamr|%wmUvms}qUNdxFs#b>u60Y(k|Ih*J3gv+C$}Y^S*!1h zS#MM=SG`)Z3TNJ?N~v{bT#awm-K&_;@*_3wfTz7U0&hFKPl(ReSJQCHR8zS`b zK-~MjzAp0!FdqOITg{F3<@Q8V+UAa`LFXPF_=*-lnewXmzV)hILe;M1k>vSQRcE@a zE2{rMUzVw;d((W=oYbc)y4NfA3l;m*75&lU8!)APFMlmTzHi=^u~a3<#PPRI-#VRc z>=vkQ!O|1c!#p`^ZuBqrC(4qJw7o5=`9N>qIDF#Wj(6wQ4qu9sZyIkJUo$63p`tlC z_|B=@r`~>ibx>$ID3lMR#wMb}8FNL-OeKa_wC^|e{j_4EsxF~hr*;d}?iJ@sV~X0F zuIi0xeq}DtR5!f&_|37KDzCGih5@(aHcWQ6f zrrmwQ_CCR}Z+YZixhp#S(PPKnCEx8|d+b8&?2TuapMAxXI4d}JB(3k%+^%`MZq+Kd z_X~~#DbILRo6*?UHI;&fCm8ow0 zpy$A=PsDo@lR{N%^75Lq<1P@oEMu+s)TlP7HVL&-^}7eBWQ3*SGeQna_7RIhG&nZZ zP$#RS|C=`RIhX8x*RD~GGOZ>d9!K1l6c96wb@cZj2A50ZZb{6MVc8ronm7rtJ&2Lx_n(e%$q5xKc2%={bxI8?X z>iT?74Z-aa3_h>@1Q^3e&;KQ}q2vv;3Q-uG9Sk(4HR!qv7@H)Bh37-I3d0P#mA3@m zbVnogYG&7#LU;u+Bqv-xQZHsG>POobp%nbgit3ecYM^y@@-)Jh2s2Ovv7lQ5Z|2EL zu-TNNB5gnxDWK*51S(P6KO(yC_z+*M*P%B!&Ng?37!g!Du zD>i$F%(?b?Cj?bCLzH#UEFvF^^w^+Dka>da&ms8<8Yc3NK*9iv7KBKSgZqIa=pFYU zZ{aqiU}+o2F1+krC})3)qCr$sb!zb3$s^~%Yfb6^dl(fW?w=h(5uzCT=MVui&IK1i zwgg`WR#YwkXATT{Fn~C3%$oy)=w&^~L4iGj8e!q`Xs#tUJqeBs9Ppx9w+S6);j`R& z4AsFR1Nakz*X%3s?zR+o^g^da`S`>ni;NAT0?5L_JGcPkJU75s0Z$0WUPKkhc^2(& z_Bj+`?%V**+H(u@zypJ4E{BfDt!!$RD|KKOKID4sfk_3G5YkJrA3}ZbYazH-Ac_zl zD-Qii=K${Gdy8pj#}D_Ub^9WRGg`y-nx&c)Fp%2j$f1l{`>n-qEWY>xy3@YC^z}HK zHrIo2SXrN--}2q^36-5oQ>%)f7=C29d-g}>sC)z1Pe*+Grfc2h7F_N-4QW@~x~o%g zb*{U51y^s{wQt=uD7b(rJ)E`;EvY{Oz+kd}`D-an%{`+za`Z!U8S+IiUmsZ-0Z<$` zEP(mnY`OmY((~6sz%_vZUhCYj+Hce?*CniJYeV!9@G};Btn=DcD9%*YBy4Y0-l`NT zcSYG)!%J6JY%52rMcUZ9ZtN9| zy=mjVsO%%K`}50u;z+u@`OZ0H-(O|dS`P{3hoT15{>h~$VLP9(a)0PibNa4$RCtSW@P|p{X@)LxsoV+8GE(7>_LLMe)MX}Q? zoxf5^qDUVE9#K*&&|@J60S7qdWhFIg_*e%|JI$ArYtaROzpE(5SAd&8KA{)-95P@# zq(eL`FYZH;={;}M2iQf2IWPZx^jY8~sE{y_;Hex|6#Gbm_aR&f$Y(5f94l!P)*C8m zGkS(KVRcvqr560AE$X3_S3!Slyc%LVuYuU{hPr^MsHT(GLTT9>`EkJb3#>{05)12! z+JLd5)Epq6#{v$ZKnaFMy7sf)lGyMJ{+|T{JP#h7ybRnUG;inTCTyfDcq1@X&Y~Wu zc=U~o3w$jrvh!L@$7^lCQt1&4D9;+ujdT^JoRO~1J!^8$TJ)sr9(to5dfS$JtAD6w z_D!mpeZw{BhSyEtEvhKRlofmrfy#xommN(b^cMGu-u{|`1|uL{)HiwuRQjQ&ak<#&*>Jdy#>$K8FV>p z3A}}Fz0)S`-G|26p7o0@S|jF}hAqwEmEL`)L#h+D7Fd;{9&}>;c$b8w2V5_LNIQ!| zNdp@gzUb~CQr_Zlr{`@&HU(!A{*z!m)Z;x;Kfvtu$Q=#q`LAF{>Vd&B(|hrX(Ia;q zlnR6Sl(*Bpc>VNAx!Jj%Y037YI1W1s^-CH4;k&JU-_7guF;u*6O2Tp$omF|41Yq)> zF3Qh4B0xS3##nq-2J@!=l8j`&4Q4mXzOb72OK|XNKJfcO!?pYQGPI80$TQfmVpoby zsPw@pNw-2D@!J_;;y&(St&}IP5Td5h#Qa@Y{bsqbQIu+oQ zkcHD#yneE2>7)h6BWDiMqiR)tVFm~Ja(tQ`;w$j;Fz?E}SK{ZcH;V2|Wa3@~59SUK zAg;n!k{7_jNTM_5t(@=j(CPDFMFiJd7JXJGYOqtSCoqe?QtHTl%hkmq`99Qwgac&Y zMeH>Gcn^yONJENjomm4g{)(-wtt}*@C~TJr>8Kpj5B6Lndxn(QVjHSHBeg*=i@zj9 z&CS?Y3sz{(-4qmrDkw_oGSYk( zzWf6xWO~ZN6GeZ9qHm$-GK!E52wQ&GD5TsbQI+GN zUq=Ne%Li=;bL zS@hi?NLx9Z3fDHyT{()4wIQc&6wKW;#y}zC1$+z{ppeoXzDJ@t5X)2naBcSoLsee_ zYlZ_4Z1O5Qk0>P*RY)ZAgDHb%CcN+sRHzM4;CVn!okPoL6dB{#MQSJdfRVZaW_$Q< zYLNk91KwK%ekJaY5SCvdEXEv`EGW%5$g1kcpE&OsI(`o9 zmb3IUNSEOL?+;>;3;D#u;=``lS&{ND0D-8OWx({Me(Lz?p~s&Z#h+S%M}Cl*ywLLg)7<(4IBSVtAO4Lm( zfHQjbi%tzrBdgR&RAU6SfcfTNvN#qIUX)!0xAdzV{NQ6Vw5_s2F}6WvjYjC3f!vzU z8CPUu;O7oz8Z5Jl8u+jmHxdn~hhT}p;6-u*%w5Rio<*|{4{%(d+3z8w@=3^$kD=U2 zlq)L7(1SzzwXI@q(R)_n1ydJ{*bIX04)}5Z8u$yavMTi7I`xfH@y4Vha%xS{2Fg}j zEf`kJRiLC*Xj8i0HAUZsxdALJnzXqOXGRZ2pTGXKrLV=kf|*L0wk0Z)!|$BFeHtvZ zE&Zuo2UnStZfH$$_e_?^v5#%8*bXqaI;-NxmOY7y`C+ z<e946Sl;mM0X;XJe2HCE~HF7YnnYDX^bh$&ZH};U+GGj_O5AqH_R1r zSK8bFvMw|<#e;8#ZieE%lxf$RrsZDKuD51y&HkG?!B8JP9(!TK3i57pcC{BA&TVPy zNc0f0$bf$uRHE$WOtM_4 z?Lbx4H=jvX3pIN(RDELaEq!uWpn6dK9XDS{?uGiF)|l<4=;2L@FgmWEUOFA`S~Jvb z7_6~FvBlVIqC9C!p1fVRYESL#-z3O`l}P@6@@8}55~wFux>K&+RqfsSyHEbK`Q4|5 zLrL~o)wrEATuMR5M#x4bWLA+|4VZ;y;@XpPqN1(YI(-wERRK%`O&o0PyXW2-yeJLSnB+RzdN0J=DE~KPs%fqnw&~a&!r~(DO=zl4D*{t zqOAUb1rG;pjz4fXUp|Jd9S0W*yWXf*_@>IAtx zv%NXlofuhBuPRsR75;7q=y&W#Vfm28W8|lH81rvyU>*hS{?9pdrM;)9J)@SV)XFnf zwW#sHwLk6mc);0#g2L>v{t`+pK?J$z)c)K6E)YY~XTU5jei;?ZK0&86=E^_=iPNZA z_EyCyeCS)@l+5xm_}0*(Dme89q(nKoyZ#d8*-^a$bXCbVxEw)ryF|6eLkGq^9#QA< zOe~;}*gPHq(*<&dBA1L|^ zMZG9Gh@wL%LIP~|V*P8BLK;qd2_Pz>D++K=p}a;EA;q|;IRO9Z!-9`F$W}s!;UB?` zK@2lQlAp@7q~bSa1ljOwLjNIQ`jF6lNa%k}*ghoe_Y5@=ZANR1s4}{;=#J|xOD(DL zfwbqn-)T)yKhCg_b(|ZlKSi@IZBep$W5P&P#B`2 U_vNk+6q;`y1#7&b63@Z^12PhRoB#j- literal 0 HcmV?d00001 diff --git a/code/MANIFEST.txt b/code/MANIFEST.txt new file mode 100644 index 0000000..23057aa --- /dev/null +++ b/code/MANIFEST.txt @@ -0,0 +1,30 @@ +# TradeAC custom-qlib-code snapshot (auto-generated) +# parent repo HEAD : fd5382caa4c4e96417a16d5a22006d9a7d625d00 +# tac-qlib/tac_qlib/contrib +# tac-qlib/tac_qlib/data +# per-file hashes (git hash-object): + 1b6298c4a5652f2e863cbdc385a1014a570fcd59 tac-qlib/tac_qlib/contrib/__init__.py + 861592c63edd6a0853a9cb174b5970435b135fc8 tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc + c76a9f17f680e74eea766eff27f7624359749ed6 tac-qlib/tac_qlib/contrib/data/__init__.py + 5c547a2ef92e075e550fe6d01508a2f1d3f536bc tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc + 4f656130d167e79dcaaeb7783a121f0b36852374 tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc + 0dd25ef161c6e0f15eafc84886e7e1381deb38c3 tac-qlib/tac_qlib/contrib/data/handler.py + b151d139a0dcde87d74b21e7c4b729176ba5c39b tac-qlib/tac_qlib/contrib/model/__init__.py + b1489f2fc0dee85f0a4f90b2e6ad545ed9c8967b tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc + 121ef237da1df1b8e21a561c3ad0db200b901339 tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc + 74d0da348cbcc3700c96b6f4fe4391488e61efc5 tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc + d3f051f3a8650c42fedc7b367b966f7c74fb5789 tac-qlib/tac_qlib/contrib/model/rank_ensemble.py + d03e6611338918d4aac5eea4adf26f85a3763652 tac-qlib/tac_qlib/contrib/model/rank_gbdt.py + 2c2f167b693f4366a769998e3c9d4804f29e31e0 tac-qlib/tac_qlib/contrib/strategy/__init__.py + c29e45e562256bf786c36f91a971b097467276e9 tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc + 6dd1c568a2961842793674390d5abffd1a0e71b8 tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc + 519a1f4c05dbe0ac018ab8b779eb33d53b4dd545 tac-qlib/tac_qlib/contrib/strategy/ic_gate.py + 79aaad9e39fcc740a773f4f63c512ce1086cfde0 tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py + 7bcee5f0b09cfa721440f1354f16f2dd9a112b12 tac-qlib/tac_qlib/contrib/strategy/regime_gate.py + fe60bacdfedd48617863be31f24b7c7daebfac5a tac-qlib/tac_qlib/contrib/strategy/weekly_rebalance.py + 92e6e90eb0cd0a25142034560f27adb6b705b1a8 tac-qlib/tac_qlib/data/__init__.py + 316bf4aa160cc8d15929ea648be03f4b4999667d tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc + 554a3f29d181b64effbf49a8161b32e7f93d8d3e tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc + 8b47f6d78ac046b6b7b2fb07bd7f3382773ffb73 tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc + 53c9007a928841fd3c3b08450f9a6520ce1ac091 tac-qlib/tac_qlib/data/config.py + 8d0644f6f0d1efb94798ed444cc73e63b643459b tac-qlib/tac_qlib/data/providers.py diff --git a/code/tac-qlib/tac_qlib/contrib/__init__.py b/code/tac-qlib/tac_qlib/contrib/__init__.py new file mode 100644 index 0000000..1b6298c --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/__init__.py @@ -0,0 +1,11 @@ +from . import data # noqa: F401 (registers tac_qlib.contrib.data) +from . import model, strategy # noqa: F401 +from .data import TACHandler # noqa: F401 +from .model import RankICLGBModel # noqa: F401 +from .strategy import OptimalStopControl # noqa: F401 + +__all__ = [ + "TACHandler", + "RankICLGBModel", + "OptimalStopControl", +] diff --git a/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..861592c63edd6a0853a9cb174b5970435b135fc8 GIT binary patch literal 374 zcmY+9y-ve06os#U@}ojKbOi=Bf&_g45E66%qyiFU@?x1LsseTrTr+g+!Yi<|vGE?f zL8OR*i4D<>iR%ak?r`*-d#-)$*KW51SbHxIH)8_u?#*A(Us#S19wCL4XfQ+&;+Y~- zK@AhE^-L4P1#g*aY620o9B5`j5fYd}8a@#b&7&m#z+I4#ny1RCgw}y6(@fX#%vt5K z=^UfZ)#30&6{*fFy56NKZcm4!_ca-PIB zSmrJF5_tJPQb$rMt)+a1C8qGbJy2COaB8x1r}JwMvW1D?fek71B6m{mR`WXE$9LcB l?7;T$sQ=S!4ds&&LK^75!A1k?4Qw{B^~L!LT|{;j^B+mFS@Qq@ literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/__init__.py b/code/tac-qlib/tac_qlib/contrib/data/__init__.py new file mode 100644 index 0000000..c76a9f1 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/__init__.py @@ -0,0 +1,3 @@ +from .handler import TACHandler + +__all__ = ["TACHandler"] diff --git a/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c547a2ef92e075e550fe6d01508a2f1d3f536bc GIT binary patch literal 216 zcmX@j%ge<81lAkcvaEpgV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##>w=j?Ny5c_}%mMMca&AwNy#TkIJ~0=L-X;}dgo;^S8`d7ehzB#0^YcoIfC4FrC5ig+@tJvui+AWXvKO%f#Q;4aHEaL? literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f656130d167e79dcaaeb7783a121f0b36852374 GIT binary patch literal 11073 zcmai4Yj7Lab>0OQZxA2=Q6xoDTD=LG57U+-S+XTl5*1sfVM(O?fa8EG>;eRZ2fYi5 zA`3Qh>Lie()Kb!n=~%AS#50xaOsdZ0hdl92ZBJ+FOq(eX(j~Hyr*576@ITs=r*Wn~ z`klL20F>!uCEmSzALqW#<2&c#-OWFp5tERWG>3d zylhGH8B5f{Q{I}kMy+@YX(3~a+E|$_ZO=HOj*K(v%($Yij63RP<@U5E!a;*>vLRmgWM%=l-r)OMLR6qS2?-;IZ#%bCia?3t-s%} z8u@4!Yq@GEH)9QNlJ2%}-00xp2c@i>R@8l>BtDZ) zofeNsd8t}9d|b?rOL_71R4OfNqKq0%$!lUl&1F!dh@+|`D}#r{v~))4bGt{!Q<{kX zRJA2OrRCAAf!r2R$>!A=aWa?6=Ed$uC-bRXRy-8m$2yD~^Pv8!l2BA78&~AoQ10oR zdM1(1O^Y*9CcRS=$HuDD>8o}e8xuhReI-ow(FcbQKPKKGPO3_8JeQfA%7a#3>ZJuH z6e&NYDw?R}RVkGm&%0~up>>FplKRY)k{1)HG#cf{)!bBa9OIoBPMxOJ4Ck@F(fil*gMaWXZjq*GZ%>>e9SWwGE`txf~+ zevFt_BsIILI1F;BbXruWvYH#4m(NZ1h+0lm&ZSV1O=9_R(8b!O;^U%3+jK9mp_8|y*A;gRS_Jf|v=oSMNvn9A7L$=qoiS7W*RjG|16wChB6DjUZ(Cl$HR{V_37 znAbg0Hk->!bcR|4-@c@hPfVrLlA6>em3Uatg<(uecN*s`rFGlr)MQ%GZKty6gC4Hh zg7MMRz$nAVQ4j*XSX=_Z(`hAkW}4`JOxzx}>h572)Ztt)3WSm!8TfvztyTiEg8=r}wUKf8*ysxOX646?NKwj+7m~ z`IT=oWtZgUtOf1}_w<__Hz(v#W;_KU4=FdE3BktSx((5TiAP3fq&Lgu$*@s<7Dd_)yCiC3-*bo)jjE} zz!xmCaK&Pdz;gxLgl~;@cG-5tGUr^i-e$9%Id|17+Y18b;w%WVy`AVgW&3^fwoZi1 z*7bM6Ss(Y}>tE7FXlU~b0hC6#uwyoQJg4TxTmlkBDxoi)gTlj`RF$cx`r_kC{7ekG zC#!*BGfciAhux{{gc2vUlNYB`SsAJhYBERaB(8<~bRQ~WP$p_VMjTHZmyan~nSIUf ze}q(*2E8VZt!`+jMOJ9GMiJUUdc{aCtMr8}YA2Sjl6g=ANOVsoCr?4@CZ+s%*sZ%$ zS}IF=EUxIyHKC0ejdWL4baJZdAqvE1x8CAEisj&{&SjfIc+$fsQAN%@2wjcVqmo}d= zOfmOYPobYqp}%S^`X*=eb!(l3+6S7w_|txc8cOO~k8oCfE4qe)FGqm~Y_JvUQQmD5a>Mb6r_%~hMUH0$17P%pp zdk$4vx0hN6uBS_1i7a!LK(Mmz#EKQ~Qfp+{&IQ_D^j-E{?<<}BdU;c9v3abrE&TV9 zzl+>RmNy(NKFmhHGP>yBa&_(oU*36drMK_eT)FrDOP={h%FVkf-oOjP7l!9M-uG_* zu$m8i@6qKh&bMxPBj*jAA6~(NxK&!jh61KVNP@oOdXa08BuJqtLC`JMiAL>1gzyw< z5;`XogbAW{+ha<)W=V1*1{||ygy&s=O3tDy0Z8_ocg~h4zG08qe$Fln&}KWJbsUgd zTfy-%_lji}?dF^X=UO_$?5Xh=I}3ITG)7&HXV;;h{bD!d6dHi2{K>h7(8}@WTB%+a zm-n0VF4#?dFv3+wuW1Nl^tIcDUDBJ_`XebVo;$0kCN>e{WD>F(;TH{{37SCMrJ%vg zu38g;xl3*4XnpdPq@!bv_=ORq#8M1n-A#?Ii*c71b2D{<|FHZ;IdrkcxOx0nWoF0&NleV-{K@J!DL(dpeaLr;Q+5V`JN(gR$L-)VXk< zIFy;p&xol6l%%=g262q30o4!?q8(Dnen#Rody6l!%&;o_ih$A|^5lrj@r`n|mL$pi z8&w+S8{B^cTx$WB)#U1eFj2G8Cg(Lu;e@kZYUWIObt?Iqy*BG+Q34s~&sz%CtN`n4 zoyBbC_yVu;7mx5!zF{AbD00WkcM`KmY9y8Ee-+k#E z5T|yaZ*`XwAfx>il0Tg1DuI@w{brzb-tnsUC2wi_o`tr({~5TqIJj)FHnrbu-7tUG zs|Q{>aBcHKN6%8nUFD9u7FzHAd3$*PW_$D-HTejl;M~w{hk9QB=ZF%Qz z*Wf|hI|q2AVT*3bOS(k^o-|f_|E#N;$xF4YHW?OF@*;KNI83{4SDA>YyHON&sszP= zE2{pHvBk`@&~*RgjHIf$Y4iMaNA)0CU)`S4sJVI<5pwG;Qb;j1mxHO#Na`6Jt*Zvp zb!$A4)SWa8>qAIPy_bfzV#aKG>VAA`#HVzQ?0LXA%=RxmJarh&4^h8&kl>_l1vyX4 zr6;f27u-9~AFlX=Oa4t|SY+3Nf7kg3E1u8`2QM64^mJDIfuidJLHtZ`+cw>5<3j6y zu>JhekNh1)YbCg0e)_6*Rhf^KgFQt@#S?ttz=Z?vdpauK;N|Sq#~0T3mc6|fjxIa6 z=FVj|=L%ffUv_R>a_lHOb}Tx&uRU?YQ|>*y)bnt;=i$YkM=DP5Ej#B8UfS{910|t- z*~+>6m;0`^Ed+O!ox8wTo#_Q-Z_NhY)A-T})k8)91{+C#4D1+Y84?WLAh2VjYIUkEIhBMIrN# z<5E@}&!wSI;qA@zDu$}e0@z7;IFqJY?^gPfeWGM~-B7gf!l7%S&Zps+VlYOw=6f?} z3)MS9o~ubD4yKzSCCpr5@&rXKmCQ#tpI{!^)P}t6u>L5`RC+f_pw#8=1@^bv_cdHUM%jDb`p%7o&ZE^DSsfI#I9JXjt%vA)i3{sVp3m*ldUx&TzN;%)%)& z)ixl39^Kee6c(KwmCkN@w^uqh7KQhn?FQf1rh~R*^4L-?V#>6XXMGLkv}QPE{!BL% z36=&3#?RqB(MSbE=y~=&l#_>Y+w<(5t{Oa@g8>78gh8wW)3vFsP0NJMvh}Be8Ht;- zG|agMn^(=*vX+Z`gnL1V13bqq3=9i6I)Pl+2{Hq>J*Gik%c(W%0FSp9gv%BJ!*L5> zI~S|L0ou9SOgUntw3m(MJdRi@^_=Fywg|&N2-XA=N zM@87Cl6BTCazeKoZnM#ukJoTOl{Kvp$XQurMPrlrGb?`@> zl~8N(;YuJ>v{yX-CC|pPXJg5;>so%P|ABJ<1Ev1qWs9S!^MmH0|8Cz^S-1X*ZC+m5 zI8fd=aDDgE-ABrIA6aM{dM|p*YS|FHWWVL+y#C_Z@9v-9{A<__M!LK2AASGeTj)IT zy6@`1jU8_X-VFb9-+bT_-8Y`Nv}Yl7wB$X? ztUOaLz@riMQFIGC)NOd_?pQUHf%t?s7JFt&Ld1_nD;%+yjPOKEJ&AVe7$pjl*EvRr zX*_0NS#kz7+Il3rh|1QH`W-yxH&LU zNg_9en+EvgG%&3*OyjYbFNOg^s9vQg=g5E8;Esxb8LTzpcR;XJ=kaSe_lRzlr5W9F zY9wBZCZO13{P8Q$w#;?O(8au?c+RA8a zqyQ z?z(g5(_!6r0)a$gdgIOqUY4WEI4YqSfFZb`6p7S5h?{`HV|1&d3&f%Gdh_k~K)SaIQ)BrV1SC>46*~)D>g_Qo+YFH7EUrUV z34rc?OiE8FLnMECb3-%6w=lL-AjK42c%7k+^PsB|WK+*lqET{=k{L=mDPbXFGNP)A z1TpGWL$Xa$k5ctlDWPDy?q=h~NH%njkvAOuX{vR_V(elj7Sq5Kgez}<46mL|bT(Ai zPc@ABBXrX~LIM|Xc?aj+Q1Wc9bnPrgDxTJoXH%tPYjOB@9sQ*{A9$N$tqUEolC$mR z?FApUcP+K=DYx&r;d@VbFI9Rnw$MIS^!>))bu-wxY~cg@R;<>LZ<(_;`QZ0$-MO@7 zPkGCp8@7cl2g=S(MfatL;pea4fn;<4bzf=I!OF(&Yj>2ozVeC9+7VoKa^fA8w$4wQ z9Id`jZ0=Tf(Q|7(=ki@}l$~wMoXh2|v~8`lZmzU+QKqYx{bJ#q~&H_Tuw%>eg8 zjaswX&$MNj%ep_^0yEq$aC1Znw`aB0LAV+1H9&S<8^YDFWrc>dc7l=}`j^<8J?1=c zJLiPvwj-toxI&b1Cq_HvEI1}w%+~dHp?0?h`0bo%t=E{j#ugXvKy=PI0#MF?xovvQ z96>RDo6h%yy)06q68g}EG+ogKPYym_yDmNY(9rOa5p@$)TVa)<8K5`wc;PB*vZ}!} zik%vvAFCwGDupD%F7+FD8G52F^enQ&1DzW&HgX@8tZqd-VD^=L- z9dDl~jXqwAK2;ulYB4_b>(Qwjp&QTqqV>HU<-KRiU1u*uZZ@^f`+jO&Z0e~D99$Y0 zDh~|39bX(cw$SxNdEnT!!S|&9Nd7YUtJJ*oYVxJzE2+}Jv0|ju^~7%uKC%#qy!n?m zHeY(?#lq#n)x(Re-Iq6Ci(emnJy{OkbK@_I_EI47yLGKU7@EJU7%vLH4YU>qEB@wr z;rY2r^ST$Em!0#0zwuO>Lod27yXOaf*o3H3unnrBIaG9i{%H$$?==7U|8za^xgLmU zq<-IMKD@{NmIKKU}BEP-4s8I{e>oRF9dM6^2M-Mm-}y1AK;}(h-<_R5@%%Sl+jmWY1dP9BUEF$KrLAjTddYcd_UF;JZEs8OIRAb3;)%!J>s`F( ziPv^tvt5&3cmCu+sdc>6JYI^)B_&agB}-o)H%e}XTIX#)*#DO8t@zv9e=)JR_n`&x zSb6V5*9We)Upw)WuF~Fz$|9z7tmHj*)7M(su>1O{>kpOM4lMW%mYfGa|J2HDewf!t z6yDw9J)gz>L z6eJBGMo?~b+^!Y3dh81FHq&K}H5BbuaKa zVytjh$8)ku#)!#>`ZDESp-xRzfErgQE&(q1CQ2C>JYZO`Gt`t4^6Is3A^8IZ1Z+0v za*%7;fZz|VkSa~VO8Wr)Z|$hGv{zcTEVb+^w;+<#MkT(EN^pI}ANp+5UYmRQEYI14 zpIVx1p%p7~%ap^FK9xCa5AfAO%6__zw{-z>+Jo>LZI2mMw^)`P0-F~&fz7d*CQ6qk z<|gZw0a^W9DiDpGMPMW`MZc9p1cbOvx8yY40e=R+@5&p$^RUAMLAYBbeX5ddsl>D@ z@wFcKLUd1cQKd+g?O&)ge;{KVj+@mR(LfJ%U``VKq6WH;-Bd9I!>$L_SBdOZO1?u0 zJ5Q1sgZ(#Cj;)r~ts|(zVoJ;FBs{{!ys54c_bD+s*h6AsC;kMnn2JzW;U72k&2-s`V!DGS?| zUA9h3iEBekds8t}5;`ls&TAWQw3fOD%fg{$o7JfHa_b{}i3@(%JaE40Q+tpP+}gE+ z-*D?Bzl{%;0-Kitca#Hn+~V+cE5gTl{xSYi*Da1^J{`7lo+JEu$Fi5(6fSYi?{gh& GrT-5_l`QlC literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/handler.py b/code/tac-qlib/tac_qlib/contrib/data/handler.py new file mode 100644 index 0000000..0dd25ef --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/handler.py @@ -0,0 +1,254 @@ +"""TACHandler: a qlib DataHandlerLP that builds datasets from the TradeAC lake. + +This is the "custom DataHandler" entry point (Option B): the handler is referenced from the +workflow yaml's ``dataset.handler`` and reads OHLCV + pre-computed ta-lib features straight +from the lake parquet files through ``QLibDataLoader`` + the tac_qlib feature provider. + +The standard qlib processor pipeline (``infer_processors`` / ``learn_processors``) still runs +on top, so existing recipes such as ``DropnaLabel``, ``CSZScoreNorm`` or ``RobustZScoreNorm`` +keep working unchanged. +""" + +from __future__ import annotations + +import os +from inspect import getfullargspec +from typing import List, Optional, Tuple, Union + +from qlib.data.dataset import processor as processor_module +from qlib.data.dataset.handler import DataHandlerLP +from qlib.utils import get_callable_kwargs + +from ...data.config import ( + LakeConfig, + timeframe_for_freq, + NON_FEATURE_COLUMNS, +) + +DEFAULT_INFER_PROCESSORS = [ + {"class": "DropAllNaN", "kwargs": {}}, + {"class": "ProcessInf", "kwargs": {}}, + {"class": "ZScoreNorm", "kwargs": {}}, + {"class": "Fillna", "kwargs": {}}, +] +DEFAULT_LEARN_PROCESSORS = [ + {"class": "DropnaLabel"}, + {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}, +] + +#: always include raw OHLCV; ta-lib columns are discovered from the lake and appended. +RAW_FEATURE_FIELDS = ("$open", "$high", "$low", "$close", "$vwap", "$volume") + +DEFAULT_LABEL = "Ref($close,-2)/Ref($close,-1)-1" + + +def check_transform_proc(proc_l, fit_start_time, fit_end_time): + """Port of ``qlib.contrib.data.handler.check_transform_proc`` (inject fit window into procs).""" + new_l = [] + for p in proc_l: + if not isinstance(p, processor_module.Processor): + klass, pkwargs = get_callable_kwargs(p, processor_module) + args = getfullargspec(klass).args + if "fit_start_time" in args and "fit_end_time" in args: + assert fit_start_time is not None and fit_end_time is not None, ( + "Make sure `fit_start_time` and `fit_end_time` are not None." + ) + pkwargs.update({"fit_start_time": fit_start_time, "fit_end_time": fit_end_time}) + proc_config = {"class": klass.__name__, "kwargs": pkwargs} + if isinstance(p, dict) and "module_path" in p: + proc_config["module_path"] = p["module_path"] + new_l.append(proc_config) + else: + new_l.append(p) + return new_l + + +def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]: + """Discover feature columns present in *every* feature file of the lake. + + Walks the `family=ta|sp` partition layout (plus any legacy flat files). + TA and SP columns are disjoint by construction, so the common set is + computed per family (columns shared by all symbol files of that family), + then the per-family results are unioned. Returns sorted field names + (without the ``$`` prefix). Empty if no features are persisted. + """ + cfg = LakeConfig(lake_root, market) + feat_dir = cfg.features_dir(timeframe) + if not feat_dir.exists(): + return [] + import pyarrow.parquet as pq + + def _family_common(fam_dir: Path) -> set: + common = None + for p in sorted(fam_dir.glob("symbol=*.parquet")): + try: + cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) + except Exception: # pragma: no cover - skip unreadable files + continue + common = cols if common is None else (common & cols) + if not common: + break + return common or set() + + common: set = set() + # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet + for fam in ("ta", "sp"): + fam_dir = feat_dir / f"family={fam}" + if fam_dir.is_dir(): + common |= _family_common(fam_dir) + # legacy flat: features/market=*/timeframe=*/symbol=*.parquet + if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists(): + pass # family layout already covered + else: + common |= _family_common(feat_dir) + return sorted(common) + + +class DropAllNaN(processor_module.Processor): + """Drop feature columns that are all-NaN over the fit window. + + The lake can hold fully-empty indicator columns (e.g. a ta-lib output that was NaN + from the start). Such columns carry no learnable signal and make ``ZScoreNorm.fit`` + warn on empty slices, so we drop them before any other processor runs. The drop set + is fixed on the fit window once (during ``fit``), then applied consistently to every + segment so train/valid/test keep identical feature columns. + """ + + def __init__(self, fit_start_time=None, fit_end_time=None): + self.fit_start_time = fit_start_time + self.fit_end_time = fit_end_time + self.cols_to_drop = [] + + def fit(self, df=None): + if df is None or len(df) == 0: + return self + window = df + if self.fit_start_time is not None and self.fit_end_time is not None: + try: + from qlib.data.dataset.utils import fetch_df_by_index + + window = fetch_df_by_index( + df, slice(self.fit_start_time, self.fit_end_time), level="datetime" + ) + except Exception: # pragma: no cover - defensive + window = df + if len(window) == 0: + return self + self.cols_to_drop = [c for c in window.columns if window[c].isna().all()] + return self + + def __call__(self, df): + if self.cols_to_drop: + return df.drop(columns=self.cols_to_drop, errors="ignore") + return df + + +class TACHandler(DataHandlerLP): + """DataHandlerLP backed by the TradeAC parquet lake. + + Parameters mirror ``Alpha158``: ``instruments``/``start_time``/``end_time``/``freq`` define + the queried window; ``feature_fields`` selects the features (default: raw OHLCV + all common + ta-lib columns found in the lake); ``label`` is a qlib expression for the target. + """ + + def __init__( + self, + instruments="all", + start_time=None, + end_time=None, + freq="day", + infer_processors=DEFAULT_INFER_PROCESSORS, + learn_processors=DEFAULT_LEARN_PROCESSORS, + fit_start_time=None, + fit_end_time=None, + process_type=DataHandlerLP.PTYPE_A, + filter_pipe=None, + feature_fields=None, + label=DEFAULT_LABEL, + lake_root=None, + market="US", + **kwargs, + ): + # default the processor fit window to the queried window (like Alpha158 without a split) + if fit_start_time is None: + fit_start_time = start_time + if fit_end_time is None: + fit_end_time = end_time + + infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time) + learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time) + + feature_fields = self._normalize_feature_fields(feature_fields, freq, lake_root, market) + if not feature_fields: + raise ValueError( + "no feature fields available for the lake; set `feature_fields` explicitly " + "(e.g. ['$close', '$rsi_14', '$sma_20'])" + ) + + label_expr, label_names = self._normalize_label(label) + + data_loader = { + "class": "QlibDataLoader", + "kwargs": { + "config": { + "feature": (feature_fields, feature_fields), + "label": (label_expr, label_names), + }, + "filter_pipe": filter_pipe, + "freq": freq, + }, + } + super().__init__( + instruments=instruments, + start_time=start_time, + end_time=end_time, + data_loader=data_loader, + infer_processors=infer_processors, + learn_processors=learn_processors, + process_type=process_type, + **kwargs, + ) + + # ------------------------------------------------------------------ config + @staticmethod + def _normalize_feature_fields(feature_fields, freq, lake_root, market) -> List[str]: + if feature_fields is None: + common = get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + feature_fields = list(RAW_FEATURE_FIELDS) + ["$" + f for f in common if "$" + f not in RAW_FEATURE_FIELDS] + elif isinstance(feature_fields, str): + feature_fields = [f.strip() for f in feature_fields.split(",") if f.strip()] + fields = [f if f.startswith("$") else "$" + f for f in feature_fields] + # de-dup while preserving order + seen, out = set(), [] + for f in fields: + if f not in seen: + seen.add(f) + out.append(f) + return out + + @staticmethod + def _normalize_label(label) -> Tuple[List[str], List[str]]: + if isinstance(label, str): + return [label], ["LABEL0"] + if isinstance(label, (list, tuple)): + if len(label) == 2 and isinstance(label[0], str): + return [label[0]], list(label[1]) if isinstance(label[1], (list, tuple)) else [label[1]] + return list(label), ["LABEL%d" % i for i in range(len(label))] + raise TypeError(f"unsupported label config: {label!r}") + + # ------------------------------------------------------------------ utils + def get_label_config(self): + return DEFAULT_LABEL + + @staticmethod + def discover_feature_fields(lake_root=None, market="US", freq="day") -> List[str]: + return get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + + +__all__ = ["TACHandler", "DropAllNaN", "get_common_feature_fields"] + + +# Make `DropAllNaN` resolvable by bare name from processor configs (e.g. the default +# ``infer_processors`` and workflow yamls that reference it without a ``module_path``), +# mirroring how qlib registers its own processors in ``qlib.data.dataset.processor``. +processor_module.DropAllNaN = DropAllNaN diff --git a/code/tac-qlib/tac_qlib/contrib/model/__init__.py b/code/tac-qlib/tac_qlib/contrib/model/__init__.py new file mode 100644 index 0000000..b151d13 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/__init__.py @@ -0,0 +1,4 @@ +from .rank_ensemble import RankICEnsembleLGBModel # noqa: F401 +from .rank_gbdt import RankICLGBModel, rankic_feval # noqa: F401 + +__all__ = ["RankICLGBModel", "rankic_feval", "RankICEnsembleLGBModel"] diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1489f2fc0dee85f0a4f90b2e6ad545ed9c8967b GIT binary patch literal 319 zcmX@j%ge<81lAkcvQ7c%#~=<2FhLogHGqui3@HpLj5!Rsj8Tk?AU0DDQ!aB9b1q91 z3nN21LkjaE#wgZG7ERWdj6mg@jJL#s67#Y>on7;aQ*)DYQhnT=eDhOMb2OQ5@j+x@ zBDZ*ofGRVSqzEXG3v!Zve0*kJW=VX! uUP0w84jZ6YX-=wL5f9J+kY9=gfW!x8Mn=ZF42Jg^j305iHL@3h!VLiOB2)PQ literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..121ef237da1df1b8e21a561c3ad0db200b901339 GIT binary patch literal 9461 zcmb_idvFv-dY{?ZmsY!af+WDe0K(D=Y4rdB76wi{ENou3!FE|Tw;Jt?q!qKfVrCX0 zWp&57sIbd%hp4lQCHP`oN#*3=sv}i-;QV7>)uryLl0PJb!`2*Eam9D3_&;I!V&7G& zlHb>}vnvTfC8>;{r)RqR>+bLQ>u>fS0|6h0YuD{R9^So<<36Vc`*K+oe*Qcv*EpH$ z-P>)@#_s6wxRyHj<>T)i&#&ZW9zo?cJP+w0BuDe3j0k6SH``Fs8BoktDC z%6iLK-K&H;|q3bQW5;3BQxTK0NOYzgs?+^KWeSKE- zV4ltqP)Nv%+SezJY7z$99Tqe+jwo7-ERE%u+%A4$k$5*u){4C2aGd zuP+EWoy4l_Hy}KuXhfLSYRPYbz=@?|zcM79jV6+sm>6K)bi7p+sDL8-tnD>IYEju9 zFKf}WSTj+y<$|??Q=~CLadarEDpo5A$>JeM`gB9;kE+q}{MaJ1MQISy(hbO9OtfZa ztZvYku9bK}TO{ey+93pzp{goGR|cDCQ8aBq8?aF74NgF9j3G^tpcRRPD*o&Fw;?Tc zus_5)Kr*1Vf6Erq1PDo0Bt5^mx}GvNJ4A5aomzPZ(mqnuwf+?f7VR zbQT*s^Bumktt}Rfw|0_Y8KgD+iF3h_coejg&^3B3iQ8L7hQ@SK%a%w2TGuMJcZesB zp7aeRbfdT>UDVM<5x>)Mu9;1%Lc_^+X(w#s&?if8V@fQcjkWY+C!L6$2 zx6z_Y1F(^V*q@B5GBkqN!3Mkx!kPls$=CrnBB|j0l7Vb(q-*5EEKdjnO)~ovp`m1* zJqoMFK!Aex)F^@Mna>;wzjs#%^1lvKdg~GEz^V9W7zhJRyR=USV9Os zAxyd-#vMx;pdO26Ql(hGEQQq$T7T!Frbt57rPv4pL;skeDQt&37xf@ZWIbr`W17MP znsTN~T)cfv84OG6$dE(>A8oQ0g+ZakizN-jCO2;DXx`D>(bloO`I%>$+dH;yT(m0N zPj`u(K3_K+eP3TwlO-Z3i4*D6oBx(CQMPz(W{vf{)KDT8 zbhApQb>rt%R4xKbDqJrQJn4{m;75<_O>+Z4mKOrOPQ@jc$ifBZt^D|2H=dmroV}hB z&i<2qcFnrWrLyaStJf?0<$&x)tt8Ez<9mH4xL}z^_*S~dThOrxCx8&Fg|{$GXQ+{Z z2;1pdZh^cAnc|DYZ-%efV@u&;k4(&B7vf4RCc%%9bO4da^C*ZPkkevVXJfe#8*Qrw z2*9HIp*>ifMG}71iuuKWt&W8t>o(|7qpS=_N!197yG04YWW;E(jR-Urfo3aSVk!M& z%tK0NrKlw)ec&_(f>@6Fr}ngtFl!Zt12oh*7HA6QRYz#%UOz05hR{q z85Eh46gpu>z^w@M(BT5D!cwe7MufbY(9MJYr5wl44?)`R@Ygs4ck4>=xAOJjg(LE< za-sDAcf}cT3~~|2sb-FwaHgEY`BRWhiSD*;>ip#u2F=Q@C-=4Izv{Rga)Lf+bK$&Q za9oNT_S$rccfHiWeymG&*c9I47tC3B3uCTVgb+aTKW>G#I)`@9kGvfI_bhb(( zBdv7kAq+zmc1~%f{QwTOvYA`8h3Aye$e6YY6vo#t?*&8l!d^xvJ%i$#^W1|_`|ZuQ zhVO^=%!Kyb(eH)!&#ruFCbWO5@{=>a82#DkKa9N{$!>VB;d;Z3;5+f@(Egc~FQuKA zeYx`LE8~~OA5=Xxz4p1;s-9_o&x5)(*+Vx1Xbt>k?S`9A-*_|ayysp0z#GVTX1sMX z-cWYi{WUE!Yg+DlL%EjLwDSl4-+x)o)x6B>jLftgVB$99Gi_ZWCt|w6UJkHuH;TDAHxw37evIx2^9S zud^(KO*P_pojbvea_5BCxlukS9Q%SyUZXjo4MI#hX9}tkHysJxtkeOtzG;Bal3tXUFYoT?X%T8ru{pN(D9&@a(()_2GrP0xcT1mw_}i#30sk&&Apzs7q zLIoVp@0b;`I;8NaUe_F`>+Uc9G#%S*xM)d>G!aq!Euc$3O+( z>P>KMqH>}tRh4oLlO->?QWZBi*$b_zU($d_*+Hzl!TlJp#olYG@>#qiaCP|v{(sxa zu^nt#xC3yp;ZbU-vQ)Y3c=Akg$&ZB(eRe{c=PtK6xiQx!t{whn6k+Ifj>q5SJ67;v z@Q8600Ld@?5H5)&`WOYi4V5yI&)!rYYc$tQ91rx3y^Cl{ui#3!bl zkaWqX2^+0c5ar2OhRH{i=)zoaE0adptC(xFjvbWF6(x4qq{L)Cjb;-?HilABNJJIL zJ(7+poTiqhA&K&$oFz^L2a)^H3$&KzeTutTobVAF7p0ugZQ+5zVgSohA*F0>S`;-> zPeO=_oDVBzIb~B8h zUJm|-pZ|AM-YHplG=aVHDe|uV;blBJ^R9fROm?uCy;c@d{IK&;9X`uzC0NGM}3ch|o zz~S3{ObcNui`k{bUrQQ#P|#Ybp^b`mDhOf)-5NzBjhwCNCg2~_0r>NIPnKpfSA^Ml z7)hT*jJ-kHQ#V}14Ri=Si!3{| zW>3iOeGOCSZ=?7Ir-O=hGZg?5(-nKC4%~0vGlPE>d(+*y@|tYrr9a9}-0uFnLw|K> zX8p6drp`(Jn)j+VyLP6cAy>WP+TN>sZ|<0S0hls|b3U$m)x%QMX8aq{J(I6w56+cwwd>N|mwP~LQpvtDxp&&Xk)aqQ zeoBF$^LR=hcZFz|CLcnB8xk6bp0i}QVu57It6TW2C|3b{)>c!)4k{1`IBg>p1mQH& zY}2VyLJEK#$3h@<(}hnm&;%bumd|)-+o_)~jI=N_Lp@k)iD@qpS)y7U^o~iA>E&ORyIIWDC0&HCS7QODKVjmUzQRt+pNS=s& zaAme@!Tnpfm_BHS&|!S_<5hv+xjv8Z@L!PpTs>F4@ov>KH&5Q*{LIYeXVRWrdHvn; zeN)c+jr(RA_obb`DJ@GISH?5r-`|^+?v}1+x^f7h^M&_cm^$h?3uy8 zibK#De`UJ+%E8RRTt)4i;0+Wt07{Tsv-x2F?;!N*RbQ4*+wM3X)U3{Sytn=O_M44U2Y&j(2QSR7 z*)dzQGu`vGz?F4AbaPGZ_iEcesJ%Xzm2Otv?EVves`hSeJC45oIXCC8n=ak-t4&Q; z%P0BC&g|x2ty(wPovRQ2RB>?cz-mQD>p}=k4jbiosIf1L+GVkTY zhRpGFcRD$#)q@_( zW39jfO;3IvcKS3OBeLnHqKPg$Menm4O0A4KgtBEi33;Ft!g} z#!E)z&W;1^hq;a34k9yVGfyeP1|84WlT&xxhVi_*~xq zsPnqo=4v>1)t3&Zt94#LZI0Faf@>@MY*Q%Lx;6LIy4=cDx$4@v@{ z!K1BinDbb5IGdKa607cmPn|Di^{-!bcwHOkI22ZhuGR*z1Y*EI66^;9H1Z2tJ-w|W zCAB{JhpcAZineZJjJ|tlBw&p)1_Zt0vZQOh^iWCQIz)(x6$G0|eCDc0-iPvM3=M}& z&V1s}JO=w^gJC`7Yw?{9++}QJ2!ib;kEWirB)Lb^K#~}*??QoG7|%a+bnwpKA=A%q y`kX8IHP`+x+=kD&>fdrt|C-zSTW;59rJJUG_2&_V&hh-l?6-f1FG9S{*Dpl(s1E?S^GpBi2^3?MWq3jaq}P8;~2_V>-+ z-ks=kla;tTH}lOmGvD{k_nP_Vni>y>=ivQsj{cyX<9$k=1{q~rn-@)o^QD?skZMLYp-^0G%ejmQ}XicoPzm~x`qW)N2 ze;q43qxG?d{svZdMH^#H{Y@ylCC{*>zghB2-ZyRi>!do#hwpl+=1s1@#loH9q}n$D zB{$r)8nf)*FkuVSi*GnM?)<4^7ZQ>j6~;v+BE~gA8xb`@7M18kkE$h-s*s2aVI`re zJ*phmB8j*d6)uVKu`|boNL2|L zIw6Eb1-}LpDNTqNU`bi&k;Dm)f(Z-Zgrdk%k=7t|&=Q0*#{=EMjgd%rM8Mz`lT1e9 z!-ABMRUw|xghWz{#3GZjFqBY&p5B|l8i~nqjnRf?vu}4#@0F86R2-9qj>|9g?309j z`+A0CQA=S)Nii-*1A;n|xUrVHG>!+;5ruFH)Q*{y9g@dE=4fOHV-$(KJGlFq=Y*ak z0t(OV6UNmbBLwEn_>`0-Ax4Y@AFhmG`QfMtqAGGq#VRU!u~QW!Q4@uvA}2+~Aje=z zJK*sMLXV({!zu^?iVO@yhX;cv0IJFwSa>6%jR-}8g8s5H0gz!OkxHUb1$k6KO<*NN zYg(3oCnjr3Bn-yHg+Un%h$T}RCJHQqric=wvN&;&jU0)OLcr8AEd~Y{8GyYbCIf>x z_T?nRGlmMt9Tp*h79jl~(Kr&<5^P{&@)#o$9~d}ja8@9BW}F4j;({ti*?7djh$a^) z!Nfl%-ee3{8LlD&z$O9(VsunU@LWy{E z;_wwEB@?+4YlMvrgz1LIN(?7>p@Za)hF!B;0$}sY5n4Pg88o|*P{xL$kgy3cMv^lo zGlqf%Boqf2e&PrdivvZWGB;?c6~L*e97>AX$N`8#I7B-+7*52&#=)Q=U&Pm0`6!28}~o`T*^!5)`Wi=i=ac2-KDjkSOncqCK99g`%=JE~D?>Jx?};WEI;` z)Dj9B_wC;OL~Ag}9Vs%a9XsLiG^U zvGSqFFgCGN)KQW=B&MPoEeyKqvaEs2j3BgO51)j3BKCqKL(oXVAoxi?fXb7MLKX#+ z;@A!&;&>#HQiP7OvL#(h!xHg`!>8mpR89B*Oj3Z? zy6P1C*jANN_vervqhmgepj%TrdZ)MM0UwwJ$YF3k^F zO3=0nXc*eHPui+sB;y%%nWggg9)GvkK4q`K#5|BP6qSB^OleD+Pun)25B*I7mt>hF z4ED6`5PgfSQx2?!!A&_U#;(2!A3Cz9V#Y4Xns!MBPKRWd9Iag1Avq*xD>wEW#&H(w zoaMSnC+b~PzmcmLv)063{-#|fjU_v+VaiChr_4|7JsFopp`Wxy&v;0e!uUocz{W&4X$(>!2O|MvrW%`9h)4$kmKYi$ zoFj>-WOzm4R4he)5L6wyFk}QHqaU^}vw&<+^o^inA{5DPX)D68Yay~&@ zcNT}y?Q$%sP3TUTT2XuU8QlR7Dn2ahw!uUqsyjegH5JocgOPYRk&0^ppYAZ2s$0P= z-9{AC`M7RRC{Uof3m@3R_?T`{hw(SAyW;YV!Du2praRdOojj}nH$sf-RyemR?KoPG z`Fycw7ihK%_PU4EKNXna-c>Z*yO`nLMdL8T9ZXK>O(90sP?7M)ER;@Q-j6>O;RcuC zKCk24zUhytKIE z{;~V&1IztWxz2;R#zO=@eJOK(rMB)?WG0ezIsY zw>ES3F$lwjOC|t6-1uWg<72$%xwPet+6u!_G6fPp*1>TKKVhj*BvlNE#1Hco5Y<$T zWt7HT^JpcbaJOix%@-#k#9QJ1U?n4NY}ud)hGxSE8a% zrSbTxKK3B~%G4Qjnctz?^oo55;pQR`#1?uuG}#i;Ab z6f;Zd+O(hglqg+CW_;`#My!K`p8#%1-&B1CmKL*b`J1j!*Zc+u!iN1KIns5eWRMw_ zEatDNhIIYt#xhQ`bhlVg+O$TqE)7^R$fKLgn)26#H&=8SW58THQ@s$?3spl-(xFcz z5{faXS_(qdvUx7X#%05c@MaL9FZwW3iZ8R<6``=uQR#el!{B*H%QI^c6iy&6G2-w; zdy9dnfk+XMK}aASLm3>3iFdJpm>hLM9g8H99Vkxg>m@hhK*_5w1=NQR3opTDsuULu zu}H*p{MouO&$x2(;7wD?Jd9La6@GWm?wbg&VAqjE7(p60aZ_5vuNBGNI~rX8Dz;2+L;bX;G5+i|W>RXjr$Zp(MScKp09I?bvrVJ z0AU}*Ws)3E#ON1Dqua=>RcZ6l>eoCMZj*R$Dk% zO=jvnYqs}Y&w}{J{(`?@#anah$jp)K(d^#$Eg!gVyBFI(?7q|e*J{4=U|u-1)OtAY zJ(4*=COj|`c>CD=nYlCXg%{f29nCid7WaL4=+2=ZTk~Cq@}9$)<16k>S$W~|;`Y3I z7u*i~8d?Z1K9hIvhBxBz-|C*}emj--Yyu?kbP!P=HRTlaBQ*R z_Nj;7Zp`7gtSfkGW^FT_vugJ8+(gbJz~}J!XSc)0YFyyoY03GvWO~0`Lw?w)nNzd9 z+1|MedEZvR)YQ!$op~kOzOdz;K+d-vo*8Dae`Y^m_QE%7x!tnZet*xM&imqn=KPiy zavNXxs4?&D&73H7^nB>PLBVI}uI1?_~ zZWK8VHo)+DhERal!;cod4r(+XQ@2hbJSc0V84xKTeMw4j9GP0=5CTE$A3#UK zSCC$C56fDJ?FXZ1YzY#?fL$RESa%{_LP-?mdHhliP?Z(A7?ozr1e=MoLar2hy^exP zuo>=AP1ADC=3LF@e2tJfS@1P3`!?l#oASP=z#GrTZ12Lei~I7PJ>a;varSa{`;vF_ zl5Mj=X<#TlMIVuWoI%y^gVr+F&*NypBJurJ$tk&}xgi`fyjjz4qeB46`X&w++;B_j zpX4=5<*($EY=Ch{HBzl)N6A^y<}a72gkuB;ATY*FzTXWV)GJ<0WAX(vn_{Tba`uX) z!%c%xWXyO(B-3Ohx0#X+eo2;qm4Ey+Q2!xcnRuD8rn&Ec0umpAT)fYJm**)#!G%h) zIKc%heRm)nx?N3SVBHl8MdA@H6w=+q>JTO60yZcwIXa|V#R$5?$W`j@P>5whLm`!j z42EV5eqNy^t~X|HE;MMOO`}n5#p~A@ZpBqIH*uhEC2p@-58T`+NzIx^`5mp~l!E?biq|}vU{;K7HB04h zX=X1=zUuu)x0j*K(m1EQ0O+0O@A6mj-K#O@JW95D&10(U(XKTO_^FiwvC-}?x8AEY z@lLtp7Esx(?J}W8cbld1H|?krQg$p?mi7-Z9A*?lCw?JT9Q(AGN{uqlYpp!f_d zh9RpNQe+uo z7MCJ${L~QI@xkEMI7Bn%_USfBx{?;f150G`25p`LN?o$A`f?!3Xuohdp@xlg_WLw6uQT z^Nwd>&j7UfKukPiVcC0?b)iq6f4VznO;oM*fILki{!dZZeYR_0A9MVfAN41=SNb}zx@m*pnpGXYYClJ1KZ(cr zN}1l(5>HhzBl+Ycb39~$!*$LngKMz=-jtk^mkNh&EwgkH;Hl8|}bLvMca zw=smO*Afe8St)kvtk)YPtf(r+40KHUO42DW6M#ihl_Y_Gi(WHSr z-}KP;Y@w-Ty7!k2&GUh|K(V>ttDSy*rgzouZuGz=?K{CQJ##L5V!3ViCvCfzE)Kv} z7FsqH8rB!qJ@uKNfa*AZ^Jn!aO*>bcxVrUO-}}`{$T47H z+bRllwsV5Jju;;SS=D?;+7{tVW%&Q)JAmap$1y`=N6=(_7slyb#JFkOyWF4HaC%Wv zh3Qhjg+9BBeOz#zotO}Ns+l0kfoRz|O0!?{NEWGd?vZv!`1f(`Fzuo*Mb>XUZR6hR z!yn?|346df`TY3=lA4B6k=X7xwh4lH1^07ESrqjT?%%PaOB5`P3J@K;Bffb zr(D8J0uG#j!kxC`s#Q$Y-7$_%c zD?(s#+h5eqoksR;&q`DC{Drv-xu)|AS3bCQ``Z0&_tkv+!3P(Yw!eVNkK~6<=RdP@ zTXubB=QbTesbRe#h$R+dF63u0U*?ZcDEl(7cqMNJ4aElu5{D^WiEucfDFa;(-SjHE()4PD2LZdPDc7$DY<+#7Qfi|QbYkoO0U;#%eoe;?%_4Lt&F_r!gLE#B&>A{Y-nQ~UdO&HnIMv?FcDl9NO%m3sgBP41_*GyiR~*tdP| z+GKB9J;8IGL0BM!B!%V;t2MjrPd}>dTD9W~`_Z~{)k!56C-khkspR3-x2}4rRL6OG z9@nGvxx_o{7x>k6m6!^R>sS5F_MS)e8&>W37LifOKt?4(_CH>f3>k5;0Qw6LuCLI2 zX+@l1=P=3-X&LNThLsrgsRWW!O4?|DzEnC;DltgPg9KWYzrly7J(Qx$#~>?qH3Sb? z0;CfZvTzES7{x)=b{1J1(YwRAo}qo|JO0z&Oh~eU5e*>W?%UgM_)GR{{d!KEvNth literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py new file mode 100644 index 0000000..d3f051f --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py @@ -0,0 +1,189 @@ +"""Seed-ensembled LightGBM that early-stops on cross-sectional RankIC. + +``RankICEnsembleLGBModel`` wraps ``RankICLGBModel`` (per-day RankIC feval + +``metric='None'`` + ``first_metric_only`` early stopping) over a seed ensemble: +one sub-model is trained per seed with identical hyper-parameters, and +predictions are averaged across seeds. This is the model class the +``tac-rd-rank-ensemble-isolated`` reference run wires into its workflow +(``module_path: tac_qlib.contrib.model.rank_ensemble``). + +The ensemble inherits the RankIC early-stopping behaviour of the single-seed +model (valid RankIC drives the stopping iteration) while the seed averaging +stabilizes the prediction against any single seed's early-stopping path. + +Training is parallelized: the seed sub-models train in a thread pool — +``lgb.train`` is C++ and releases the GIL, so concurrent seeds do not block on +the GIL (5 seeds ~40min/5 on this box). Measured on a 6-physical-core / 12 SMT +host: the seeds scale ~2x, not linearly — the runs are memory-bandwidth bound +and each Booster caps its threads at ``cores // workers`` so 5 concurrent +boosters don't oversubscribe; larger-core hosts scale better. The qlib data +pipeline is warmed once on the calling thread (fills the handler cache), and +each worker then prepares its **own** ``lgb.Dataset`` (independent handle, so +no concurrent ``construct()`` on a shared handle — LightGBM's ``Dataset`` is +not thread-safe to build). qlib's ``R`` recorder is also not thread-safe, so +the per-seed evaluation curves are logged on the calling thread after the pool +finishes. + +Wired into a workflow yaml like: + + model: + class: RankICEnsembleLGBModel + module_path: tac_qlib.contrib.model.rank_ensemble + kwargs: + loss: mse + learning_rate: 0.02 + num_leaves: 31 + n_estimators: 3000 + num_boost_round: 3000 + early_stopping_rounds: 200 + min_data_in_leaf: 20 + lambda_l2: 0.5 + colsample_bytree: 0.8 + subsample: 0.8 + subsample_freq: 1 + reg_alpha: 0.1 + reg_lambda: 1.0 + seeds: "42,7,2026,99,123" + parallel: 5 + +Any ``**kwargs`` other than ``seeds``/``parallel`` are forwarded unchanged to +every ``RankICLGBModel`` sub-model (same params, different ``seed``). +""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from typing import List, Optional + +import pandas as pd + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP + +from tac_qlib.contrib.model.rank_gbdt import RankICLGBModel + +__all__ = ["RankICEnsembleLGBModel"] + + +class RankICEnsembleLGBModel(RankICLGBModel): + """Seed ensemble of RankIC-early-stopping LightGBM models. + + Parameters + ---------- + seeds : comma-separated integers, one sub-model per seed. + parallel : number of seeds to train concurrently. ``0`` (default) = auto + (all seeds, bounded by the available cores); ``1`` = sequential. + **kwargs : forwarded to every ``RankICLGBModel`` sub-model (model + hyper-parameters). ``seeds``/``parallel`` are consumed here and not + forwarded. + """ + + def __init__(self, seeds: str = "42", parallel: int = 0, **kwargs): + self.seeds = [int(s.strip()) for s in str(seeds).split(",") if s.strip()] + if not self.seeds: + raise ValueError("seeds must contain at least one integer") + self.parallel = int(parallel) + # drop seed/parallel handling from the base kwargs, keep everything else + self._model_kwargs = dict(kwargs) + super().__init__(**self._model_kwargs) + self._models: List[RankICLGBModel] = [] + + # --------------------------------------------------------------- helpers + @staticmethod + def _cores() -> int: + try: + return max(1, len(os.sched_getaffinity(0))) + except AttributeError: + return max(1, os.cpu_count() or 1) + + def _worker_count(self) -> int: + if self.parallel > 0: + return min(len(self.seeds), self.parallel) + return min(len(self.seeds), self._cores()) + + # ------------------------------------------------------------------ fit + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + """Train one RankICLGBModel per seed and keep them for prediction. + + The qlib data pipeline is warmed once on this thread (handler cache), + then each seed sub-model trains in a parallel worker thread on its own + ``lgb.Dataset`` (LightGBM releases the GIL in ``lgb.train``). Evals + are logged on this thread after the pool (qlib's ``R`` is not + thread-safe). + """ + n_round = num_boost_round or self.num_boost_round + n_es = early_stopping_rounds or self.early_stopping_rounds + + if len(self.seeds) == 1: + m = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + m.fit( + dataset, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + evals_result=evals_result, + reweighter=reweighter, + **kwargs, + ) + self._models = [m] + return + + # Warm the qlib handler cache once on this thread so the workers' + # concurrent prepare() calls only hit cached frames (no first-write race). + proto = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + proto._prepare_data(dataset, reweighter) + + workers = self._worker_count() + # Cap per-Booster threads so concurrent seeds don't oversubscribe + # (LightGBM's num_threads=0 uses ALL cores per Booster). + per_booster = max(1, self._cores() // workers) + + def fit_seed(seed): + m = RankICLGBModel(seed=seed, **self._model_kwargs) + if workers > 1 and "num_threads" not in m.params: + m.params["num_threads"] = per_booster + ds_l = m._prepare_data(dataset, reweighter) + booster, evals, names = m._train_from_datasets( + ds_l, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + **kwargs, + ) + m.model = booster + return m, evals, names + + with ThreadPoolExecutor(max_workers=workers) as ex: + results = list(ex.map(fit_seed, self.seeds)) + + self._models = [m for m, _, _ in results] + + # Merge + log evals on the main thread (qlib's R is not thread-safe). + if evals_result is not None: + for m, evals, names in results: + for k in names: + for key, val in evals.get(k, {}).items(): + evals_result.setdefault(f"{k}.seed{m.params['seed']}", {})[key] = val + for m, evals, names in results: + self._log_evals(evals, names, prefix=f"seed{m.params['seed']}.") + + # -------------------------------------------------------------- predict + def predict(self, dataset: DatasetH, segment="test") -> pd.Series: + """Average the per-seed predictions over the given segment.""" + if not self._models: + raise ValueError("model is not fitted yet!") + preds = [m.predict(dataset, segment=segment) for m in self._models] + if len(preds) == 1: + return preds[0] + frame = pd.concat(preds, axis=1) + return frame.mean(axis=1) diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py new file mode 100644 index 0000000..d03e661 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py @@ -0,0 +1,238 @@ +"""LGBModel variant that early-stops on cross-sectional RankIC instead of l2. + +Standard qlib ``LGBModel`` early-stops on the regression loss (mse). For +cross-sectional alpha signals the quantity we actually care about is the per-day +rank correlation (Rank IC), which mse early-stopping does not optimize for. +Experiments on the 50-ETF lake (SP-5d 55-feature panel) show that early-stopping +on a custom RankIC feval lifts RankIC 0.047 -> 0.075 vs. the mse-stopped model. + +This class reuses ``LGBModel``'s data preparation but: + + - tags each ``lgb.Dataset`` with per-day query ``group`` sizes so a ranking + metric can be computed per trading day; + - injects a custom ``feval`` (mean per-day Spearman of pred vs label) into + ``lgb.train``; early stopping then selects the iteration that maximizes + RankIC on the valid set; + - forces ``metric='None'`` + ``first_metric_only=True`` so early-stopping + tracks RankIC only (not the regression loss). + +Wired into a workflow yaml like: + + model: + class: RankICLGBModel + module_path: tac_qlib.contrib.model.rank_gbdt + kwargs: + loss: mse + learning_rate: 0.03 + num_leaves: 31 + n_estimators: 500 + ... + +The rank feval is used for early-stopping selection only; the objective stays +the configured loss (default mse). Set ``rank_eval=False`` to fall back to the +plain LGBModel behaviour (early-stop on the loss). + +Generic: works for any cross-sectional panel whose qlib dataset index has a +``datetime`` level (each level value = one query group). The per-day groups are +derived automatically, so no universe-specific configuration is needed. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +import numpy as np +import pandas as pd +import lightgbm as lgb + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP +from qlib.contrib.model.gbdt import LGBModel +from qlib.workflow import R + +__all__ = ["RankICLGBModel", "rankic_feval"] + + +def _group_averaged_rank(values: np.ndarray, gid: np.ndarray, offs: np.ndarray) -> np.ndarray: + """Averaged (tie-corrected) rank of ``values`` within each group, vectorized. + + ``gid`` maps each row to its group id; ``offs`` holds the cumulative row + offsets so that group ``i`` occupies rows ``[offs[i], offs[i+1])``. Returns + the same result as ``pandas.Series.rank(method='average')`` applied per + group, but in one pass (``np.lexsort`` is the only non-linear step). + """ + n = len(values) + order = np.lexsort((values, gid)) + ord_rank = np.empty(n, dtype=np.float64) + ord_rank[order] = np.arange(n, dtype=np.float64) - offs[gid[order]] + 1.0 + sg = gid[order] + sv = values[order] + newblock = np.empty(n, dtype=bool) + newblock[0] = True + newblock[1:] = (sg[1:] != sg[:-1]) | (sv[1:] != sv[:-1]) + blockid = np.cumsum(newblock) - 1 + block_mean = np.bincount(blockid, weights=ord_rank[order]) / np.bincount(blockid) + out = np.empty(n) + out[order] = block_mean[blockid] + return out + + +def _per_day_spearman(preds: np.ndarray, labels: np.ndarray, group: np.ndarray) -> float: + """Mean per-day Spearman rank correlation of preds vs labels. + + ``group`` holds the number of rows of each trading day (query group), in + order. Days with <3 valid rows or a constant pred/label are skipped. + + Vectorized: per-day Spearman == Pearson of the per-day rank transforms, + and the Pearson moments (``sum``, ``sum`` of products/squares) aggregate + over each day with ``np.bincount``. Runs ~10x faster than the per-day + ``pd.Series.rank()`` loop that preceded it — this feval is invoked on the + train and valid panels every boosting round, per seed. + """ + if group is None or len(group) == 0: + return 0.0 + offs = np.concatenate([[0], np.cumsum(group.astype(int))]) + gid = np.repeat(np.arange(len(group)), group.astype(int)) + rp = _group_averaged_rank(preds, gid, offs) + rl = _group_averaged_rank(labels, gid, offs) + n_g = group.astype(float) + s_p = np.bincount(gid, weights=rp) + s_l = np.bincount(gid, weights=rl) + s_pl = np.bincount(gid, weights=rp * rl) + s_pp = np.bincount(gid, weights=rp * rp) + s_ll = np.bincount(gid, weights=rl * rl) + cov = n_g * s_pl - s_p * s_l + var_p = n_g * s_pp - s_p ** 2 + var_l = n_g * s_ll - s_l ** 2 + denom = np.sqrt(var_p * var_l) + valid = (n_g >= 3) & (denom > 0) + corr = np.where(valid, cov / np.where(denom == 0, 1, denom), 0.0) + return float(corr[valid].mean()) if valid.any() else 0.0 + + +def rankic_feval(preds, dataset): + """LightGBM feval: mean RankIC (higher is better in lgb convention).""" + labels = dataset.get_label() + group = dataset.get_group() + ric = _per_day_spearman(preds, labels, group) + return "rankic", ric, True # (name, value, higher_is_better) + + +class RankICLGBModel(LGBModel): + """LGBModel that early-stops on per-day RankIC via a custom feval.""" + + def __init__(self, rank_eval: bool = True, **kwargs): + super().__init__(**kwargs) + self.rank_eval = rank_eval + + def _prepare_data(self, dataset: DatasetH, reweighter=None) -> List[Tuple[lgb.Dataset, str]]: + ds_l = [] + assert "train" in dataset.segments + for key in ["train", "valid"]: + if key in dataset.segments: + df = dataset.prepare(key, col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + if df.empty: + raise ValueError("Empty data from dataset, please check your dataset config.") + x, y = df["feature"], df["label"] + if y.values.ndim == 2 and y.values.shape[1] == 1: + y = np.squeeze(y.values) + else: + raise ValueError("LightGBM doesn't support multi-label training") + + if reweighter is None: + w = None + elif hasattr(reweighter, "reweight"): + w = reweighter.reweight(df) + else: + raise ValueError("Unsupported reweighter type.") + + # per-day query groups: each trading day is one group + if self.rank_eval and isinstance(df.index, pd.MultiIndex) and "datetime" in df.index.names: + group = df.groupby(level="datetime").size().to_numpy(dtype=np.int32) + else: + group = None + + d = lgb.Dataset(x.values, label=y, weight=w, group=group, free_raw_data=False) + ds_l.append((d, key)) + return ds_l + + def _train_from_datasets( + self, + ds_l: List[Tuple[lgb.Dataset, str]], + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + **kwargs, + ) -> Tuple[lgb.Booster, dict, List[str]]: + """Train a Booster from already-prepared ``lgb.Dataset`` objects. + + Pure training — no ``R.log_metrics`` — so it can be called from worker + threads (qlib's ``R`` recorder is not thread-safe; the caller decides + when/where to log). Returns ``(booster, evals_result, segment_names)``. + """ + if evals_result is None: + evals_result = {} + ds, names = list(zip(*ds_l)) + + callbacks = [ + lgb.early_stopping( + self.early_stopping_rounds if early_stopping_rounds is None else early_stopping_rounds + ), + lgb.log_evaluation(period=verbose_eval), + lgb.record_evaluation(evals_result), + ] + if self.rank_eval: + # early-stopping must be driven ONLY by the RankIC feval, not l2. + # metric='None' suppresses the default l2 metric; first_metric_only + # makes early_stopping track the single remaining (rankic) metric. + self.params["metric"] = "None" + self.params["first_metric_only"] = True + feval = rankic_feval + else: + self.params.pop("metric", None) + self.params.pop("first_metric_only", None) + feval = None + + booster = lgb.train( + self.params, + ds[0], + num_boost_round=self.num_boost_round if num_boost_round is None else num_boost_round, + valid_sets=ds, + valid_names=names, + feval=feval, + callbacks=callbacks, + **kwargs, + ) + return booster, evals_result, list(names) + + def _log_evals(self, evals_result, names: List[str], prefix: str = "") -> None: + """Log recorded evaluation curves to qlib's active recorder.""" + for k in names: + for key, val in evals_result.get(k, {}).items(): + name = f"{prefix}{key}.{k}" + for epoch, m in enumerate(val): + R.log_metrics(**{name.replace("@", "_"): m}, step=epoch) + + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + if evals_result is None: + evals_result = {} + ds_l = self._prepare_data(dataset, reweighter) + self.model, evals_result, names = self._train_from_datasets( + ds_l, + num_boost_round=num_boost_round, + early_stopping_rounds=early_stopping_rounds, + verbose_eval=verbose_eval, + evals_result=evals_result, + **kwargs, + ) + self._log_evals(evals_result, names) diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py new file mode 100644 index 0000000..2c2f167 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py @@ -0,0 +1,11 @@ +from .ic_gate import ICGateTopkDropoutStrategy # noqa: F401 +from .optimal_stop import OptimalStopControl # noqa: F401 +from .regime_gate import RegimeGateTopkDropoutStrategy # noqa: F401 +from .weekly_rebalance import WeeklyRebalanceDropoutStrategy # noqa: F401 + +__all__ = [ + "ICGateTopkDropoutStrategy", + "OptimalStopControl", + "RegimeGateTopkDropoutStrategy", + "WeeklyRebalanceDropoutStrategy", +] diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c29e45e562256bf786c36f91a971b097467276e9 GIT binary patch literal 234 zcmX@j%ge<81lAkcvSNYsV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##=)E1tpoei8;X~`327Tc_l^pIYrDsDL+l-TRiy?x%gtBBv|wodwhIiPELIM zN`}uMTYeeqCl(awmn0_Z7UpCoff(^%Ml#4Kpg?g+QDRAIdZm7Rd}dx|NqoFsLFFwD ro80`A(wtPgB2J(gAcqtS0ErLGjEszT8FU_S%U|G9Xk;&92MPiJzWF`Z literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6dd1c568a2961842793674390d5abffd1a0e71b8 GIT binary patch literal 10428 zcmcIqYit|Wm7d{4;+qs7qTU=?4~y2zmhCuk{E{szajHmZByVCSS&HI}BuadFW)xfM z5O52(rBSC*nk=kjvl|6lph{zO6<~L(0tM<|i#9<2a7?FUW?ifoMX?ALyHM&BE&6NE zxx*o8Md@Q7*YcfnUuVue=iKk!IhQ}P+btBNM>oH=@Usqz`Xv^$WK;?I{#77uP%IUq zSen(v>4Yw%qmkCf^$A1BKxjkU7^;CfW4tzD3YkbR4OI&DujA))DgRsK+SQc@=tb_ILHF6d~WTA?S)cWT2>tv=~c7 z;{Ae{N~dGVIcAUnGCCg-#8|XHno5d%Dh^eAMC9g{nb{Q2MER5;^b1^6jHQwhXpGIl zLm04F&O+~HCMGZuMjM$<#beRsqfjF(CuV?ED$OOaLpmkIkO7lQ#+R8(^IVdNiNa3i z7L_@Z=J+U=6k~CYuooGSBQleEkz=N(p`Kq3r=#NZG}99s;0AyZ_DUr=CN;~5^Be<{ z>*piMMUJ&3BMD9jGLa-pTGA)Et%zIl|OV! z?bIH>6z6cV^IV)Q4?5|Zr_B!Mf|`vb zmm)-YFGk`^WR~-(IE&_s36^AviKmitFp*0zxp^{8J|-*%`i&lGXBnH%-P?)ER*|mqB&Y)5(9RR z+{GpE;Y(a>ZeA?EypLH*#$$_M{dvwZ1;!lVQ|Z(abchJ^{h;GiR8`_Uw*>kDe}|Vo z6XW4kv80${F7iuBRN>+*7(!qOt4MMgxXIFzz&Q&X`r%Wc?crqPX)c=L+39Hu zC}B1>7aU-ojPZPm7l;j_Ko*Y7KSo7V6Tp)LwZ)n>Nx{vM|Xwo1}tn-VB2xM?n71z%Z`hegchU>;<#~(A$ZSCHMu`WgtBj;Q`-Z zg$tzEUrwZmh{hacVCgf-rNj)d1AGKGjV|##plEqP0m>n>GO*&9Sn{)P21f@ts6aCh z>yWG;SUnR@ogHU-_74mk2$r~&a159|fR3;k02d7(3CajN86N}RQ&CXm^#FJw!w3&> zUIwrp6~FpR_{9*+Es5~}&ZrZMC6*G52A~LsfKaf}sY3M=Km!El5Nlx^QR@Fqn-Ak(oH6(=4-J zWjD^_0+^d6>Ay|}ZAy*0VJZf+wPG%tP%&v5RO}CE4~o^utC+NjDLNspSaIVBv)mld zaf;(X{7|fM?nN#hjwL~I#THE^5~-wGiqfUkV1uU#{#z+ZU4(x{-=IXTmAdATsF&ar z!s;YivPf3R#?o&ZB>T7NZ_+d+=_D$Qi<=}#4<&uMWPlQkRFh@6Y>iWe? zogNe;(SFH0`Lj?vpiT5)=pf8M_PZ<8r+X-~ z<65n3YS^Y~%$B03vDh3eHgpslyFM{P#kPfV`hIOi%0c-Wvxe*DV&fioIEoDnJnY5R zeObdDQ&ag_s&;#`hL>vZ!&|7b!(ai84FBo--9X%+Bx<4TFxadnmcFTbw^R>Usvn<% z9YqhPK-`D$tcTkW0TKawF+2k~C=_~xv3aFsx9MxG7of9%HoZdKb9!H1xwf*#W>@mg zj-083Z-@G5*)O3`7FdJ=2uv@)b7xv8`);L$rL`$!bdv5$9gHk{@$S9EiYHf3UpKB8 z30edVib>nc75yA1DlW*-@X^I^B#~N5ii!~qG!ZeV=Ud?&ptF>~_W&iJfZ$%LrflI7 zb|40=1a?zaRfiXF{9(vIUoOuL+bgz$i;-Q-x3Jg{gnsb|tR{My2JFzU0P3*16@WN|puOjCLgLT_e;BT`+%}*r9cR{!p_s zNj8nE_GD`oM_@_N*W1-o*)bS$eXoIsP;03?wLT-%D*Zx; zqKKc^GA@v$x6(_aB%5TuS*uwfQwKWe)7mP}g~1A~Q7{q!Pu#DS7lt&d^1Nx%;4tHs z++Q|IkgNN}^$E$efZNxO1m{t6SsIUo@*s5WNQ=)?@JYw-Cy^H1&tURT9&G+~L$-Z!=(xOq79z=_~`NdhSZ{;b~OO~5fO}8-8 zvG7=>Rii2^fCfLk{ukgGPRWLfhSCcL3UKC}*0=IxZ8zBM#;bM zL}ipl!RlNBn>JM;Q=>Kx^fspTt31)lJOqoA&q+;E!;5gvxxVLf{I;p$x4>(>!b79b zCt&<(X#JvAS9t<|Tuo1FH49T3Re5S_2~dKQ8h5o=lQtikx_$|?*ifaT@?2MHK?}`N zvrvV@lKlZ3wk$kTd9Oxk$dYN1oPQX35Cu!P?SNje7x1nsY~95hlFq93U9dN^ry579WN0tT03Ijpx%t0D5Aep< zr9D-$>>?}K^>tEPlK%fhn(My+Picfz{u9t+6L`#Sh_%~J_9v{ps-3Ju>iDunf*f}; z!6Xf=P|ITz)*}Dl^ z9^9RGvC98}<^M@)+ZBh(dkf&u2L`Hg-UFOzTS3XIPJKbw_{SJAAn}9w*sK`TRX$aC zJgzG64U%4-d+9&_as9VH**t-9hnWb^6kVEC%p(yoGK#@r#Snv=4@D0*13W-4#bcC3G2z%*2;eFfhyf+= zS}Yt^Of1Kxak82`<bPjrf2fS`jBu8$Ie;#8%?*h3UU59)Uw z8Ow4J_?E`t0uQhWI-NW_HpcHmfk;5ji_}5PNRSPW`HB^8nt&BtuRu_)24Y{JJjFsd zsdf4Y%PLj@g(o*-0tk|1xyw~?yHm&#KT#C@>GK!VdEfaxoP}U)idUL= zkee+SFxk7stpjvvus2th(uFuJW2s1Jj3(~7ASvKIK`*w+eC$GEB+0eX^a#;}-zc(!6# zA6{3m-JA>t*pcu);Em(Isq11xh zU|B)2ig5cAS9Ps8)!J}5fNYsrTGj;8gXoH#jm^$h+HLCkfOo}VD6S1~)yw=S5;dy+ z1$0jGUH?ZjHAdr~vN{Hb28VkN2+1FF>9g%%U z^1es1mSTHXq5Y8DeyGrXT5dmm>uA1xTsF03t*aM`fsR68NDd4Y0;lA_sas9C$*El6 zR6cNCHnn6etB(V}8_&J+T*22X`+7H5K6pIm>&^SlXDxTFzSV_q@A*#O+kJPegT=;< zw*s#RzFW63yy2A_2eMBTt-hSK_1m8H(BB0%hi|tZzU$euKC~|8J$*S_-#v%><>a;G zS}5=6$?A&@Ero_&xuLhva7b=Aly7(_YrpGnU+>QQg9ZPP>>t`ZR5&;)9~{m5AIsYA zd3;$@(aRLPgR*z<=Mb73x$AVV9xHhE%bxw4-n{3byz|IbXWn_D*xtWtTJwI(v2CP$ zLlF4w?9N*6`kP)|%G!#~x~!qt*#1`A>urU`;I(tvk=2f(Cs6PV%AUbZ!&ZOJGnn^0 zo*k)*t{j#{@eQfyYJkAmjU%rd zSv$1u$$PqhmA`pSECde7@aI2p+aKFxzd!fh+%5XOMJU{2f0}qdk(+uZcj1Nn=wIe$ zW(zYh8U99N*~c*eCl>rc*&n>^3~o$qj(q>g_ny4nKm7i)ck3J1`U*`)#M>usI}Z?jLZGWt_H?dKZuD+0|G4L; zeed_(@l0Tot?uQqYh!Dk4{fd69?ILXT~B#h-xSsl|MiOO4rXgVa@M00KDuehdk*Jp zhrw3J-Odwt*E_Fa1dT%ogQ^S^m;O)3PA$$?{AQ~AK@eDj%G ztla#BYny29e{P!b@)x=Tjn>+KRo=Sqdz#BJ$KjTTN7`d*|g>Rj(^Zw4D@WC z1FP4a%nsjkdJ4`xvU5+tIUqX+HZFb$3Jvt!Jp1?O-Z^(WaQKy`tw#&4aoIKg!NohS zFOpDjhwSPoxCUewbl*Cba}9jxdVJdt^S<9gc^g64I$xo#Q?BbQ)E$)T4i+0*Hk!AV z?@T>kn3|TSra?vSL$EO1j_hgB1319S+SBVpZ#=u<-njT(KTgNjxz5YJV0N?$xDLub zjTLaHM-KIl>?m-kZ_>c+jO;seYxINByl*l)ijx*L9@<)j`TWD#@1J|`56;=uR8V8w z?H{^2w_9N%pE8syu=b^mkQ_L&J?4|3&D4q<$aPSO3t$>HyQ)IU! zzds-up1XwaLH4_=)F(Qt(XX33{s|P7BWp{7;kgBKZn}6d+>NHK& zxPPsqjh*-PKy4GM#%OH0?=Tqm+_%*j=jq?NXKCX})vN0Hmq2o~*6b3P9<0Sf1Rc&0 z;xhiEjW>`&8h-K>5%?#uV1MAJGwK1f{(-vk&uPe6ksOqjF4CiPxjLX-t`4X-?r=g= zeczX4!`3Pb<^ literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/ic_gate.py b/code/tac-qlib/tac_qlib/contrib/strategy/ic_gate.py new file mode 100644 index 0000000..519a1f4 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/ic_gate.py @@ -0,0 +1,117 @@ +"""Realized-IC circuit breaker TopkDropout strategy. + +Subclass of ``qlib.contrib.strategy.signal_strategy.TopkDropoutStrategy`` that +holds the book (issues NO orders) while the streaming realized RankIC of the +deployed signal is below threshold — i.e. the model's cross-sectional +predictions are no longer earning against realized forward returns. When the +gate is open it behaves exactly like the reference TopkDropoutStrategy. + +The gate is evaluated per trade step on the trailing mean realized RankIC of +the signal over the last ``ic_window`` trading days whose label is fully +realized as of the decision date (no lookahead — a 5d fwd label ``close[t+6]/ +close[t+1]-1`` is only known at ``t+6``). + +Two wiring modes: + +* ``ic_gate``: a precomputed ``pd.Series`` indexed by datetime of booleans + (True = gate open / trade allowed). Computed once by the caller (e.g. + ``rd_backtest``) and looked up per step. Missing dates default to open. +* realized-IC self-computation: when ``ic_min_rankic`` is given but no + ``ic_gate``, the strategy computes the per-date realized RankIC itself from + ``self.signal`` (the pred scores) and the lake 1d bars via + ``tac_qlib.risk_limits.realized_rankic_series``, then applies the same + trailing-window comparison. Works when instantiated from a workflow YAML + PortAnaRecord config (``lake_root`` / ``market`` must be provided). +""" + +from __future__ import annotations + +import pandas as pd + +from qlib.backtest.decision import TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import TopkDropoutStrategy + +from tac_qlib.risk_limits import ic_circuit_breaker, realized_rankic_series + +__all__ = ["ICGateTopkDropoutStrategy"] + + +class ICGateTopkDropoutStrategy(TopkDropoutStrategy): + """TopkDropout with a streaming realized-IC circuit breaker. + + Parameters + ---------- + topk, n_drop, method_sell, method_buy, hold_thresh, only_tradable, + forbid_all_trade_at_limit : same as ``TopkDropoutStrategy``. + ic_min_rankic : float — pause new trading while trailing realized RankIC is + below this threshold (0 disables the gate). + ic_window : int — trailing window for the realized RankIC mean (default 22). + ic_label_horizon : int — label horizon in trading days (default 6). + ic_min_obs : int — min realized labels before the gate arms (default 10). + ic_gate : pd.Series, optional — precomputed per-date gate (bool indexed by + datetime). When provided, it overrides self-computation. + lake_root, market : str — lake location for self-computed realized IC. + """ + + def __init__( + self, + *, + topk, + n_drop, + ic_min_rankic: float = 0.0, + ic_window: int = 22, + ic_label_horizon: int = 6, + ic_min_obs: int = 10, + ic_gate=None, + lake_root: str = "", + market: str = "US", + **kwargs, + ): + super().__init__(topk=topk, n_drop=n_drop, **kwargs) + self.ic_min_rankic = float(ic_min_rankic or 0.0) + self.ic_window = int(ic_window or 22) + self.ic_label_horizon = int(ic_label_horizon or 6) + self.ic_min_obs = int(ic_min_obs or 10) + self._ic_gate = ic_gate + self._realized_ic = None + self.lake_root = lake_root or "" + self.market = market or "US" + + def _load_realized_ic(self): + if self._realized_ic is None: + pred_start_time, pred_end_time = self.trade_calendar.get_step_time( + self.trade_calendar.get_trade_step(), shift=-self.ic_label_horizon + ) + pred = self.signal.get_signal(start_time=pred_start_time, end_time=pred_end_time) + if isinstance(pred, pd.DataFrame): + pred = pred.iloc[:, 0] + self._realized_ic = realized_rankic_series( + pred, self.lake_root, self.market, label_horizon=self.ic_label_horizon + ) + return self._realized_ic + + def _gate_open(self, trade_start_time) -> bool: + ts = pd.Timestamp(trade_start_time) + if self._ic_gate is not None: + # precomputed gate series: look up the latest known decision date <= ts + known = self._ic_gate[self._ic_gate.index <= ts] + if len(known): + return bool(known.iloc[-1]) + return True + if self.ic_min_rankic <= 0: + return True + realized = self._load_realized_ic() + limits = { + "ic_min_rankic": self.ic_min_rankic, + "ic_window": self.ic_window, + "ic_min_obs": self.ic_min_obs, + } + tripped, _reason, _trail = ic_circuit_breaker(realized, ts, limits) + return not tripped + + def generate_trade_decision(self, execute_result=None): + trade_step = self.trade_calendar.get_trade_step() + trade_start_time, _ = self.trade_calendar.get_step_time(trade_step) + if not self._gate_open(trade_start_time): + return TradeDecisionWO([], self) + return super().generate_trade_decision(execute_result) \ No newline at end of file diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py new file mode 100644 index 0000000..79aaad9 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py @@ -0,0 +1,217 @@ +"""Optimal-stopping / stochastic-control strategy for cross-sectional signals. + +Entry is a control policy: a symbol opens a position only when its cross-sectional +signal percentile is at or above ``entry_pct`` (i.e. it is one of the top-ranked +names) and the portfolio has fewer than ``topk`` open positions. + +Exit is an optimal-stopping rule: a held position is stopped (closed) when its +signal percentile falls below ``exit_pct`` (the continuation value of holding is +no longer worth the risk), OR after ``max_hold_days`` (time stop / finite +horizon), OR when the position P&L breaches ``sl`` (loss control) and the +position has been held at least ``min_hold_days``. + +Sizing is fixed ``notional`` per position (equal-weight control), unlike the +TopkDropout cash-allocation heuristic. + +Wired into qrun workflows like any ``BaseStrategy`` (see ``PortAnaRecord`` +config). Mirrors the API usage of qlib's ``TopkDropoutStrategy``: ``Order``/ +``OrderDir`` from ``qlib.backtest.decision``, ``trade_calendar`` / +``trade_exchange`` / ``trade_position`` injected by the backtest executor. +""" + +from __future__ import annotations + +from typing import List + +import pandas as pd + +from qlib.backtest import Order +from qlib.backtest.decision import OrderDir, TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import BaseSignalStrategy + +__all__ = ["OptimalStopControl"] + +DEFAULT_NOTIONAL = 20_000.0 +DEFAULT_ENTRY_PCT = 0.80 +DEFAULT_EXIT_PCT = 0.50 +DEFAULT_MAX_HOLD_DAYS = 10 +DEFAULT_MIN_HOLD_DAYS = 2 +DEFAULT_SL = -0.06 + + +class OptimalStopControl(BaseSignalStrategy): + """Optimal-stopping long-only strategy over a cross-sectional signal. + + Parameters + ---------- + topk : max number of concurrent positions. + entry_pct : min cross-sectional score percentile required to OPEN (0..1). + exit_pct : held positions are stopped when score percentile < exit_pct. + max_hold_days : hard time stop (finite-horizon close). + min_hold_days : minimum holding days before stop-loss is evaluated. + notional : $ per position (equal-weight control). + sl : stop-loss threshold as fraction of entry price (<= 0), disabled if 0. + """ + + def __init__( + self, + *, + signal=None, + topk: int = 10, + entry_pct: float = DEFAULT_ENTRY_PCT, + exit_pct: float = DEFAULT_EXIT_PCT, + max_hold_days: int = DEFAULT_MAX_HOLD_DAYS, + min_hold_days: int = DEFAULT_MIN_HOLD_DAYS, + notional: float = DEFAULT_NOTIONAL, + sl: float = DEFAULT_SL, + risk_degree: float = 0.95, + trade_exchange=None, + level_infra=None, + common_infra=None, + **kwargs, + ): + super().__init__( + signal=signal, + trade_exchange=trade_exchange, + level_infra=level_infra, + common_infra=common_infra, + **kwargs, + ) + self.topk = topk + self.entry_pct = entry_pct + self.exit_pct = exit_pct + self.max_hold_days = max_hold_days + self.min_hold_days = min_hold_days + self.notional = notional + self.sl = sl + + # ------------------------------------------------------------------ utils + @staticmethod + def _pct_rank(score: pd.Series) -> pd.Series: + return score.rank(pct=True) + + def _entry_price(self, pos) -> float: + # Position stores avg entry price under key "price" (see Position.position) + price = pos.position.get("price") + if price is None: + price = pos.get_stock_amount("price") + return float(price) + + def _pnl_pct(self, pos, mark: float) -> float: + entry = self._entry_price(pos) + if not entry or entry != entry: + return 0.0 + return mark / entry - 1.0 + + def _is_tradable(self, code, start, end, direction) -> bool: + try: + return self.trade_exchange.is_stock_tradable( + stock_id=code, start_time=start, end_time=end, direction=direction + ) + except TypeError: # some exchanges take no direction kwarg + return self.trade_exchange.is_stock_tradable(stock_id=code, start_time=start, end_time=end) + + # ------------------------------------------------------------ decision + def generate_trade_decision(self, execute_result=None): + trade_step = self.trade_calendar.get_trade_step() + trade_start, trade_end = self.trade_calendar.get_step_time(trade_step) + pred_start, pred_end = self.trade_calendar.get_step_time(trade_step, shift=1) + pred_score = self.signal.get_signal(start_time=pred_start, end_time=pred_end) + if isinstance(pred_score, pd.DataFrame): + pred_score = pred_score.iloc[:, 0] + if pred_score is None or len(pred_score) == 0: + return TradeDecisionWO([], self) + + pct = self._pct_rank(pred_score) + time_per_step = self.trade_calendar.get_freq() + current_temp = __import__("copy").deepcopy(self.trade_position) + + holdings = {} + for code in current_temp.get_stock_list(): + if abs(current_temp.get_stock_amount(code)) > 1e-6: + holdings[code] = current_temp + + # ---- optimal stopping: close held positions ----------------------- + sell_orders: List[Order] = [] + closed_today = set() + kept = {} + for code, pos in holdings.items(): + held = current_temp.get_stock_count(code, bar=time_per_step) + mark = self.trade_exchange.get_deal_price( + stock_id=code, start_time=trade_start, end_time=trade_end, direction=Order.SELL + ) + if mark is None or mark != mark: + continue + rank = pct.get(code, 0.0) + stop_pnl = held >= self.min_hold_days and self.sl < 0 and self._pnl_pct(pos, mark) <= self.sl + if held >= self.max_hold_days or rank < self.exit_pct or stop_pnl: + amt = abs(current_temp.get_stock_amount(code)) + o = Order(stock_id=code, amount=amt, start_time=trade_start, + end_time=trade_end, direction=Order.SELL) + if self.trade_exchange.check_order(o): + sell_orders.append(o) + self.trade_exchange.deal_order(o, position=current_temp) + closed_today.add(code) + else: + kept[code] = mark + + # ---- equal-weight control: target notional per name ----------------- + # candidate opens: top-ranked names whose signal pct >= entry_pct + rank_desc = pred_score.sort_values(ascending=False) + held_codes = set(kept) + opens = [] + for sym in rank_desc.index: + if len(opens) >= self.topk: + break + if sym in held_codes: + continue + if pct.get(sym, 0.0) < self.entry_pct: + continue + if not self._is_tradable(sym, trade_start, trade_end, OrderDir.BUY): + continue + opens.append(sym) + + targets = held_codes | set(opens) + if not targets: + return TradeDecisionWO(sell_orders, self) + + # total value (cash + marked positions) -> per-target notional + total_value = current_temp.get_cash() + for code, mark in kept.items(): + total_value += abs(current_temp.get_stock_amount(code)) * mark + + target_notional = total_value * self.risk_degree / max(1, len(targets)) + + # ---- rebalance kept positions toward target weight ------------------ + buy_orders: List[Order] = [] + for code, mark in kept.items(): + cur = abs(current_temp.get_stock_amount(code)) * mark + diff_notional = target_notional - cur + if abs(diff_notional) / target_notional < 0.02: + continue # skip tiny rebalances + amount_delta = diff_notional / mark + direction = Order.BUY if amount_delta > 0 else Order.SELL + o = Order(stock_id=code, amount=abs(amount_delta), start_time=trade_start, + end_time=trade_end, direction=direction) + if self.trade_exchange.check_order(o): + (buy_orders if direction == Order.BUY else sell_orders).append(o) + self.trade_exchange.deal_order(o, position=current_temp) + + # ---- open new positions at target weight ---------------------------- + for sym in opens: + px = self.trade_exchange.get_deal_price( + stock_id=sym, start_time=trade_start, end_time=trade_end, direction=OrderDir.BUY + ) + if px is None or px != px or px <= 0: + continue + amount = target_notional / px + factor = self.trade_exchange.get_factor( + stock_id=sym, start_time=trade_start, end_time=trade_end + ) + amount = self.trade_exchange.round_amount_by_trade_unit(amount, factor) + o = Order(stock_id=sym, amount=amount, start_time=trade_start, + end_time=trade_end, direction=Order.BUY) + if self.trade_exchange.check_order(o): + buy_orders.append(o) + + return TradeDecisionWO(sell_orders + buy_orders, self) diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/regime_gate.py b/code/tac-qlib/tac_qlib/contrib/strategy/regime_gate.py new file mode 100644 index 0000000..7bcee5f --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/regime_gate.py @@ -0,0 +1,215 @@ +"""Regime-gate TopkDropout strategy. + +Subclass of ``qlib.contrib.strategy.signal_strategy.TopkDropoutStrategy`` that +holds the book (issues NO orders) while a regime detector says the market is in +an unfavorable state. When the gate is open it behaves exactly like the +reference TopkDropoutStrategy. + +Three detector types are supported (all causal — no lookahead): + +* ``dispersion``: cross-sectional standard deviation of 22-day rolling returns + across the universe. Gate closes when CS dispersion < threshold (low + dispersion means the spread between winners and losers is too narrow for + TopkDropout to exploit). +* ``vol``: cross-sectional mean of 22-day rolling realized volatility. Gate + closes when avg vol is outside a band ``[vol_low, vol_high]`` (strategy + needs moderate vol — too calm or too turbulent both hurt). +* ``hmm``: pre-computed HMM posterior for regime 1 (``sp_hmm_p_regime1``). + Gate closes when posterior < threshold (model is not confident the calm + regime is active). + +The gate is provided as a precomputed ``pd.Series`` of booleans indexed by +datetime (True = trade allowed). The companion ``compute_regime_gate`` +function builds this series from lake bars; call it once before backtesting +and pass the result as the ``regime_gate`` parameter. +""" + +from __future__ import annotations + +import pandas as pd + +from qlib.backtest.decision import TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import TopkDropoutStrategy + +__all__ = ["RegimeGateTopkDropoutStrategy", "compute_regime_gate"] + + +class RegimeGateTopkDropoutStrategy(TopkDropoutStrategy): + """TopkDropout with a regime-gate circuit breaker. + + Parameters + ---------- + topk, n_drop, method_sell, method_buy, hold_thresh, only_tradable, + forbid_all_trade_at_limit : same as ``TopkDropoutStrategy``. + regime_gate : pd.Series — precomputed per-date gate (bool indexed by + datetime). True = trade allowed, False = no orders. Missing dates + default to open (trade allowed). + """ + + def __init__(self, *, regime_gate=None, **kwargs): + super().__init__(**kwargs) + self._regime_gate = regime_gate + + def _gate_open(self, trade_start_time) -> bool: + if self._regime_gate is None: + return True + ts = pd.Timestamp(trade_start_time) + known = self._regime_gate[self._regime_gate.index <= ts] + if len(known): + return bool(known.iloc[-1]) + return True # default open if no history yet + + def generate_trade_decision(self, execute_result=None): + trade_step = self.trade_calendar.get_trade_step() + trade_start_time, _ = self.trade_calendar.get_step_time(trade_step) + if not self._gate_open(trade_start_time): + return TradeDecisionWO([], self) + return super().generate_trade_decision(execute_result) + + +# --------------------------------------------------------------------------- +# Precomputation helper +# --------------------------------------------------------------------------- + +def compute_regime_gate( + detector: str, + threshold: float = 0.0, + *, + lake_root: str = "", + market: str = "US", + start: str = "2015-01-03", + end: str = "2026-08-19", + vol_low: float = 0.0, + vol_high: float = 999.0, + hmm_field: str = "sp_hmm_p_regime1", +) -> pd.Series: + """Build a per-date regime gate series from lake bars. + + Parameters + ---------- + detector : str — ``"dispersion"``, ``"vol"``, or ``"hmm"``. + threshold : float — for ``dispersion``: min CS dispersion to allow trading. + For ``hmm``: min HMM posterior to allow trading. + Ignored for ``vol`` (uses ``vol_low``/``vol_high`` band instead). + lake_root, market : str — lake location. + start, end : str — date window. + vol_low, vol_high : float — annualized vol band for the ``vol`` detector. + hmm_field : str — HMM feature column name for the ``hmm`` detector. + + Returns + ------- + pd.Series — bool, indexed by datetime. True = trade allowed. + """ + from tac_qlib.data.config import LakeConfig, resolve_lake_root + + cfg = LakeConfig(resolve_lake_root(lake_root or None), market) + symbols = _universe_symbols(cfg) + close_df, vol_df = _load_daily_bars(symbols, cfg, start, end) + if close_df.empty: + return pd.Series(dtype=bool) + + if detector == "dispersion": + return _dispersion_gate(close_df, threshold) + elif detector == "vol": + return _vol_gate(close_df, vol_low, vol_high) + elif detector == "hmm": + return _hmm_gate(cfg, symbols, threshold, start, end, hmm_field) + else: + raise ValueError(f"Unknown detector: {detector!r}") + + +def _universe_symbols(cfg) -> list: + """Read symbols from the lake symbols.parquet.""" + import pathlib + + sp = cfg.lake_root / "symbols.parquet" + if sp.exists(): + df = pd.read_parquet(sp) + col = "symbol" if "symbol" in df.columns else df.columns[0] + return sorted(df[col].astype(str).str.upper().tolist()) + return [] + + +def _load_daily_bars(symbols, cfg, start, end): + """Load daily close prices for all symbols into a wide DataFrame.""" + closes = {} + vols = {} + for sym in symbols: + p = cfg.bar_path("1d", sym) + if not p.exists(): + continue + try: + df = pd.read_parquet(p) + except Exception: + continue + if not len(df): + continue + tcol = df["t"] if "t" in df.columns else df["date"] + ts = pd.to_datetime(tcol) + df = df.assign(_t=ts).set_index("_t").sort_index() + df = df.loc[start:end] + if len(df) < 22: + continue + closes[sym] = df["c"] + if "v" in df.columns: + vols[sym] = df["v"] + close_df = pd.DataFrame(closes) + vol_df = pd.DataFrame(vols) if vols else None + return close_df, vol_df + + +def _dispersion_gate(close_df, threshold): + """Cross-sectional dispersion of 22-day rolling returns.""" + if close_df.empty or close_df.shape[1] < 2: + return pd.Series(dtype=bool) + ret = close_df.pct_change(22) + cs_disp = ret.std(axis=1) + gate = cs_disp >= threshold + gate.iloc[:22] = True # warmup: allow trading + return gate + + +def _vol_gate(close_df, vol_low, vol_high): + """Cross-sectional mean of 22-day rolling realized vol.""" + if close_df.empty or close_df.shape[1] < 2: + return pd.Series(dtype=bool) + import numpy as np + log_ret = np.log(close_df / close_df.shift(1)) + rv22 = log_ret.rolling(22).std() * (252 ** 0.5) + cs_mean_vol = rv22.mean(axis=1) + gate = (cs_mean_vol >= vol_low) & (cs_mean_vol <= vol_high) + gate.iloc[:22] = True # warmup + return gate + + +def _hmm_gate(cfg, symbols, threshold, start, end, hmm_field): + """HMM regime posterior gate from persisted SP features.""" + feat_root = cfg.lake_root / "features" + all_posteriors = {} + for sym in symbols: + # check both ta and sp family paths + for family in ("sp", "ta"): + p = feat_root / f"market=US" / f"timeframe=1d" / f"family={family}" / f"symbol={sym}.parquet" + if not p.exists(): + continue + try: + df = pd.read_parquet(p) + except Exception: + continue + if hmm_field not in df.columns: + continue + tcol = df["t"] if "t" in df.columns else df["date"] + ts = pd.to_datetime(tcol) + s = pd.Series(df[hmm_field].values, index=ts, name=sym) + s = s.loc[start:end].dropna() + if len(s) > 0: + all_posteriors[sym] = s + break + if not all_posteriors: + # no HMM features found — default open + idx = pd.date_range(start, end, freq="B") + return pd.Series(True, index=idx) + post_df = pd.DataFrame(all_posteriors) + cs_mean = post_df.mean(axis=1) + gate = cs_mean >= threshold + return gate diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/weekly_rebalance.py b/code/tac-qlib/tac_qlib/contrib/strategy/weekly_rebalance.py new file mode 100644 index 0000000..fe60bac --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/weekly_rebalance.py @@ -0,0 +1,202 @@ +"""Weekly-rebalance TopkDropout strategy. + +Turnover-reduction variant of ``qlib.contrib.strategy.signal_strategy.TopkDropoutStrategy``: +the topk/n_drop selection and sizing are identical to the reference, but the +target book is recomputed only on the first trading day of each ISO week; on the +other days the strategy issues NO orders (holds the book untouched). + +The weekly cadence is derived from the qlib trade calendar: a rebalance happens +when the current trade step's date belongs to a different ISO ``(year, week)`` +than the previous trade step. ``hold_band_pct`` (default 0) optionally skips +tiny rebalances: when a name's existing position differs from the new target by +less than this fraction, no order is generated for it. +""" + +from __future__ import annotations + +from typing import List + +import numpy as np +import pandas as pd + +from qlib.backtest import Order +from qlib.backtest.decision import OrderDir, TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import TopkDropoutStrategy + +__all__ = ["WeeklyRebalanceDropoutStrategy"] + +DEFAULT_HOLD_BAND_PCT = 0.0 + + +class WeeklyRebalanceDropoutStrategy(TopkDropoutStrategy): + """TopkDropout rebalanced once per ISO week; holds otherwise. + + Parameters + ---------- + topk, n_drop, method_sell, method_buy, hold_thresh, only_tradable, + forbid_all_trade_at_limit : same as ``TopkDropoutStrategy``. + hold_band_pct : skip order for a name whose deviation from target weight is + below this fraction of the target (no-trade buffer band). + """ + + def __init__(self, *, topk, n_drop, hold_band_pct: float = DEFAULT_HOLD_BAND_PCT, **kwargs): + super().__init__(topk=topk, n_drop=n_drop, **kwargs) + self.hold_band_pct = hold_band_pct + + @staticmethod + def _iso_week(ts) -> tuple: + return (ts.year, ts.week) + + def generate_trade_decision(self, execute_result=None): + import copy + + trade_step = self.trade_calendar.get_trade_step() + trade_start_time, trade_end_time = self.trade_calendar.get_step_time(trade_step) + + cur_week = self._iso_week(trade_start_time) + prev_week = getattr(self, "_last_week", None) + self._last_week = cur_week + + if prev_week is not None and prev_week == cur_week: + # not the first trading day of this ISO week -> hold + return TradeDecisionWO([], self) + + pred_start_time, pred_end_time = self.trade_calendar.get_step_time(trade_step, shift=1) + pred_score = self.signal.get_signal(start_time=pred_start_time, end_time=pred_end_time) + if isinstance(pred_score, pd.DataFrame): + pred_score = pred_score.iloc[:, 0] + if pred_score is None: + return TradeDecisionWO([], self) + + if self.only_tradable: + + def get_first_n(li, n, reverse=False): + cur_n = 0 + res = [] + for si in reversed(li) if reverse else li: + if self.trade_exchange.is_stock_tradable( + stock_id=si, start_time=trade_start_time, end_time=trade_end_time + ): + res.append(si) + cur_n += 1 + if cur_n >= n: + break + return res[::-1] if reverse else res + + def get_last_n(li, n): + return get_first_n(li, n, reverse=True) + + def filter_stock(li): + return [ + si + for si in li + if self.trade_exchange.is_stock_tradable( + stock_id=si, start_time=trade_start_time, end_time=trade_end_time + ) + ] + + else: + + def get_first_n(li, n): + return list(li)[:n] + + def get_last_n(li, n): + return list(li)[-n:] + + def filter_stock(li): + return li + + current_temp: "object" = copy.deepcopy(self.trade_position) + sell_order_list: List[Order] = [] + buy_order_list: List[Order] = [] + cash = current_temp.get_cash() + current_stock_list = current_temp.get_stock_list() + last = pred_score.reindex(current_stock_list).sort_values(ascending=False).index + + if self.method_buy == "top": + today = get_first_n( + pred_score[~pred_score.index.isin(last)].sort_values(ascending=False).index, + self.n_drop + self.topk - len(last), + ) + elif self.method_buy == "random": + topk_candi = get_first_n(pred_score.sort_values(ascending=False).index, self.topk) + candi = list(filter(lambda x: x not in last, topk_candi)) + n = self.n_drop + self.topk - len(last) + try: + today = np.random.choice(candi, n, replace=False) + except ValueError: + today = candi + else: + raise NotImplementedError(f"This type of input is not supported") + + comb = pred_score.reindex(last.union(pd.Index(today))).sort_values(ascending=False).index + + if self.method_sell == "bottom": + sell = last[last.isin(get_last_n(comb, self.n_drop))] + elif self.method_sell == "random": + candi = filter_stock(last) + try: + sell = pd.Index(np.random.choice(candi, self.n_drop, replace=False) if len(last) else []) + except ValueError: + sell = candi + else: + raise NotImplementedError(f"This type of input is not supported") + + buy = today[: len(sell) + self.topk - len(last)] + for code in current_stock_list: + if not self.trade_exchange.is_stock_tradable( + stock_id=code, + start_time=trade_start_time, + end_time=trade_end_time, + direction=None if self.forbid_all_trade_at_limit else OrderDir.SELL, + ): + continue + if code in sell: + time_per_step = self.trade_calendar.get_freq() + if current_temp.get_stock_count(code, bar=time_per_step) < self.hold_thresh: + continue + sell_amount = current_temp.get_stock_amount(code=code) + sell_order = Order( + stock_id=code, + amount=sell_amount, + start_time=trade_start_time, + end_time=trade_end_time, + direction=Order.SELL, + ) + if self.trade_exchange.check_order(sell_order): + sell_order_list.append(sell_order) + trade_val, trade_cost, trade_price = self.trade_exchange.deal_order( + sell_order, position=current_temp + ) + cash += trade_val - trade_cost + + if len(buy) == 0: + return TradeDecisionWO(sell_order_list, self) + + value = cash * self.risk_degree / len(buy) + for code in buy: + if not self.trade_exchange.is_stock_tradable( + stock_id=code, + start_time=trade_start_time, + end_time=trade_end_time, + direction=None if self.forbid_all_trade_at_limit else OrderDir.BUY, + ): + continue + buy_price = self.trade_exchange.get_deal_price( + stock_id=code, start_time=trade_start_time, end_time=trade_end_time, direction=OrderDir.BUY + ) + buy_amount = value / buy_price + factor = self.trade_exchange.get_factor( + stock_id=code, start_time=trade_start_time, end_time=trade_end_time + ) + buy_amount = self.trade_exchange.round_amount_by_trade_unit(buy_amount, factor) + buy_order = Order( + stock_id=code, + amount=buy_amount, + start_time=trade_start_time, + end_time=trade_end_time, + direction=Order.BUY, + ) + buy_order_list.append(buy_order) + + return TradeDecisionWO(sell_order_list + buy_order_list, self) \ No newline at end of file diff --git a/code/tac-qlib/tac_qlib/data/__init__.py b/code/tac-qlib/tac_qlib/data/__init__.py new file mode 100644 index 0000000..92e6e90 --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/__init__.py @@ -0,0 +1,25 @@ +from .config import ( + LakeConfig, + BAR_FIELD_MAP, + FREQ_TO_TIMEFRAME, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, + resolve_lake_root, +) +from .providers import ( + LakeCalendarProvider, + LakeInstrumentProvider, + LakeFeatureProvider, +) + +__all__ = [ + "LakeConfig", + "BAR_FIELD_MAP", + "FREQ_TO_TIMEFRAME", + "UNKNOWN_FIELD_NAMES", + "timeframe_for_freq", + "resolve_lake_root", + "LakeCalendarProvider", + "LakeInstrumentProvider", + "LakeFeatureProvider", +] diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..316bf4aa160cc8d15929ea648be03f4b4999667d GIT binary patch literal 522 zcmaKozfQw25XS8!O`4QS&?06wlmYqxAVh&ys%TRvAS5p*LtGTFlj5e8I}+hj z2XEk&iH$!JNKDvOVPN2fpU$7}KA-QgTn>0IsAk*OWwD1a*JXVV~zwS zSXjgoHnD|698n+z;Sv{{dy9LbNQ%NIz6eNw(K(9kJ4{LwCoDW`e-yeZXwyr!lcoLS zT$Q)$J!rNX?Oo{9_tiqP*EoQqE*!Nwjb^XjX{hRPv=?wL?|O6T$kD3I_d5 zLSM2gwIJCr<6{O~+k?!qJhau~^exI+8dJG1vvCqLsg{h|N{6`|36|!+de!JnM)Q$m zKVs-hThA>WgMxu;;29JRd;_yYS<;BYIpv;BKlgv1@&~hVhJP%S2S7OoxJU2W5q+)H z=wMLG>DlTPPfkq&(E1- literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..554a3f29d181b64effbf49a8161b32e7f93d8d3e GIT binary patch literal 11464 zcmc&)Yfv0lcJA)!c|T#8VR%S1){BvN2$0M}4|@dYu|h}!B*)m=rbRa}XkPAaK#T{k zvgIVBvK)b}6p*z`IGdEA?22(VDXa1-ZOT+c8bGxT! z287TiyH#nJdr#lTx%ZxX?>XN+xBuC0w_xxbTlxJlvI@g~iC(nFq!RcBhGW3bp zN@!xuKwFg1%G!XovknNItP8>-wirS;TLPg+3B7D7&_1>d!g96(!b&CF#qI{W3WWnU zmg4Me^>OGr`y9K6-OKJ{_p=A!ujW1DfJ4dkE4c>;oa~_i7kikkeUlt0VjI~ew(d>+ zKrw-x!PxpYF}C3&LY2UPdrIeT9%pqJ*3UC67wDiu%ngnjibjTmBjY?H2BQ&blncc; zUZ}TN`bRmnKv19bS^AFv2wm zH-e#%&>*6wAQd$=IOuP;SSWb*_6IuX?!fsjy7OFbgOc>sz4yMlcMJZhFvH*A#1mKg zw2U7@jj!(A%VvwgFgMIIVeUjDTUY||&6g6U!q=mr6M?{`?s~1Os02KFsEcR2JFZgK z8D40o#9E4FM6Q;K)>5OjRH&93s--4ssR)FVTY5IkG2%GS2@NWv?ZQ1&Fv13h7?6rk zOHBq@jv9{gVFsERX2QYH6eR{ZKFfFWz_uMXY9l|YoyM+~)P$%uc0yztww1&lG;Sfx z&_~s%>2NK@3qiW^NG(-gUtiEaZIJm+WLXQbomj`{=s0~4n8)bRa2Qc}wpt&Dm=G6X z8Gh>oX-z^5{e%S+jEuk}O$ih%&Fu2nGBZQb365t*xE)!c+BK?_YjcvURpsNSLQz<$ zEKFI1jZO+#{@S8b3kf^0hpsY$02?CA2qMhz*J#)IaoNg5B2iJjxXB$K1jaUN=qLJ5V`H>oL2ka9#NkBrLXB*)!=Vv|rzA@$*CL>z^b3$me6q2mtJ z%~u4Ghp80#YjV-)-mc4Z|3$k0+=Z^wy@3l|G9H!jQ5g@(_>hcG$i&2?-zbyO7zf=R z4UUY;x=<9F7z#xNPBu(LL*rpiCMPDDm~3GtMrbA+9gm2z{ZycrK7FpMyOX{Uxa22g z!!R=h+ekJHjWUrDPPQnzgdUFsMcF2d3(x>d4~Li$zh1^clxQ2v26cXAb8doVgF1Ob zp#O6ZvKOET@r^$PVh$5?c?s;JEIkJKTc3}z3DbB27fCI50_P8iM(vHo6WEUkmJrQa zifGMIxjI@aTKpsIeNv-;Sin1hmqFz|%&&`Iy1>Lh6TcP;UZ;k64zzD%Xo{-g>PPB9 z^}@m#9IOQbtryU=lo*A=iiXdLB3HPPg%b5~?|4KQkHwQmx{ZOQTcS#6zEC6 zLDs!FwxsAlKc|X%8l8#Fo#!dDULaxo`;srxB=F4FDFDp6C5RKHl2?~#deCj84Cr6 zf+EH6Bh*P)JB4O>aFD8rM8Rr>Lf4t08-9QUNHITA2;jlNgJA{~Ic(Ocg9oXvZrwu- z0V;qjmV@aA)hu#g`hXqF4Kw2*QDGIJ!a+d*<=()BW8xGwfmRd6GeLn9C;^tQQVPmw z5Cv8#7nun1(MXt!h*X3FrB4Y#up%OGk41wK5wHpjhnj>kU^9X|hg8UvYS3g`;rQYU zg>R^@pr3#EC+35fzbb!b?bWwUn~!Kz^|3sX{&{8b+gGu0dJW%I2$KqeC}1 z-cLoLt6(0O2*q57*%%i&z)<2S;00<5)HT>2u!mGV46xpBl8LAw8?%Fzbt4?8yk5|f zfPIwh#v?_-AA>-)KrcZ0;{wOaMgS*hqL*>mj9RBrX9c9tDE<^GT$C+9=W_XrkbWM1 zLO%qzuyqG!DoPoBYbD;f@6CR1u4lGq@uXDJlr;a`TC(OSUndCrfejsrwh{?j$%5Hj z_jEJo-gQXUs#WX3wDq85tzESqNn4Ld*5^~?^L!^T9)e%)fz5>;d?OCw9L8Y-IG}QZ z#Q}x805j_V!IEq-O9H0V1A5j&Y=CzI#71~GLTqA7SQEr%z}04mEf8BEwnA)$*v5KU z8^m_Dl(j?b0L<-x*vXc$PKaGoguh(2bc2*S6a*aaM3z|tz|fw!4dvDs=n=%5-Umb zP)-EZY=i(Tf&p`ttU9T*lUgj3oQGD7f2(_N`fseOt>@CM=cLy2sg~|XXH)f8(#|U> z)0HPBfuJ#T0^hg?>R}Vu&aeC2UVyUMN7HxHx)E%eWbsby8m#GQJt&NA@K(XjA`*H{ zWw5%04)zVH?we^t!Z4Nt>RLN_EG6J~niAw#4wL0e3ug-fr%pF=5=PhrH(*JuzxM=)($1I<%hVV@Hs*V{VN@K2DffaE-B+kF6h(YAVz;jg{m}XjCV5*D^$mU_-<; zFYvP@EMuiwUOr}R+Q?v*JPFGzamPnOooNg71Gqvx(-`XNmM3BP*sj%_Hr?@(*qv8l zW=wrh7gy>}Vtz;bi$EwuW%mXcbU33%G?TBYSupn;51y?N1rZMB{40Zi=vWIv#^}+~ zh1ekSkg47>fx3>|K4DG}nQ*M0>gB+T1ndC{s=!xqXHk=*Tm&#^j+fS(VF0)SV0G{m z0Q}%a7&!I8u~&q{9RNpBLFmu~V07dxi3rfH0*L+fa1J9xMka(<+?ee!KM3;S83?{c z=7~oJyN5ptp{xgH08|MXlS!H#j{$TbqTp#5h10u97#|)E-sA*63Pk~48nZI;+Y8hU z(mt|5>8T(a;J^nbbeV{;vJHVKur!LxP&RkMq3<+u;el%ayaakAS&m0%EZGEAgvlUy zAxM^w#{AYT1`N)ZQ(^*Eik0=;O^^k@A6N=tFfnCv)IbFT zcI57%HDwqt)PtLH1jOM*5Y;X0I}_$Cp4tD_ja5fg+EFDrs*}1*kvnPlSMPJl(;3i_ za13=-FZv}{?Mmqb?*mh+A&_Ul`F1?x^3K_3 z?eAT`A6<<=2skAobs$Fw?=8n%EfA?g{R=s9*Bq!cFHedZO){2a~ zY`%K7EoH0B6qU~7vt>y`##1pLm_POIK(Y(SxmRajUEH_SvGUTx#LTObrymAYec;30 zf3jQhoc>_$x>C3As$DC#jaak3SS4o`I?LZi zK_>*8<~#?OcY|o32ADN7g*&8Gl=zlDDbTIa`6Fo}YjC_p-wKM4V-YeAfXQzP;zC}#)R zAVx!Aa{yH;=c(UCr3iL-GoG|5+A`Z}yl}}DX8aM<6_Dk)g?(vrt=e{_ZM!5}RT9st z!aMPd+mk%EW_RADZ`1RaC3{tp$XMO;P50Xu+SAs3YPQoebA8@78(Q`H(_a6_dn9lD zs<$caZCdrVroF9_w|&(cNP7d4w?lGvrc9krVo|g#6h#vve~m>4*ledoH-qyQ^RpJ6 zPMgMQQq1}B@<@ySIza;FbV9$jJb4cyU`W@NOn5~I?I|^LxH>_gQgPT|Pbrz}otVRL z`E4@hd9qb)acX?g)_BUu63`|fx~&gODs}V)o7K0q4D?se>QOmOdSew&meKk$ZFnXx z1;zeayo_*0*!A?b0H5!WJ4}q=eaO`T*U(#^1h_J^%h>5>tStVgEXf)}u5i(`2?w+~ zwr?vpkdTfwjo=@fb3Dz@kb-&az>{29lO`2)UdH@SmeJ&_mHdt7;#s}0D9ZLX?Zt0l z7FjEr%RQKwGZT9;-iH_NcjdE#&GOP@HBG*Z&ER()PhvN9FJqIq-xlvdmwo620T4Y> zBltmGzfuEd0$!wp0$e|%`z#>**TD6zdK*=}3b#F=+ckr$IeY3AZ=T}jQZ&2*;HxaEQzJDt>tjt;YRq6Tbd5HwbeB?)ipQzJ^b5HYKw)C z&0RN#IOTehpF-KlWy_yK0dg-EmO1Ufbawo9=a;t{lvb-+v3D}$CvG-Nn$ zye8|!>mdP6P9FEGus@5iXfy-DCUd2VTFtl*HHDvo(#)Ex-D%ry$wq;>vfA#p-fn%n zeVx$R_pW(L=jd5_@v`JOFhjtllFL2QbZ0W-DTV*JtFu?<&q|(sOC_IFELWsGM>2rZ zCuS$+Z%CyFmxAfimTz>pr|rvZ>ZnwD;N#|{iI0x09y*>rbX+=gGF^J|8$IE95&uR< zdW)e23szD(!{2MklvmzAuyA0pdg0(q$6Dp?``ruOOYWtYmv>8*t*ezM(v>Hq$`@w3 z)+%=0KfZ8$Nw;)%*)CN)zglrDU2#mRI5E?i+4bCgdV!X9)vu7BIX-nrhhI!p22vHL zQYELrMBlw|`@+0`@v3C2&5im_`!8%YUly0l)y>u|;!7P%M`!A!;>KhLx+>_N?Ot?$ zSpGrzQpb<0R*KTq&8yXI>FPGA`j}L5e6=KyE(u5_9m%fj@nli=q2&Y1QuB|UD@2-V zUZvX7RGUN{libHw-GQ__Ah|o>jN>Us5|%vsm&i{X%MPigHC57|Ff@k<{u(Cy zH-(}6Ulr)!|0@K=_;f&p=ewIyk5u1vRxaI9RXfFZB+^#wS6ZTQsAJosklhtzwjh z8$S*;1Q1!2%@Op?Pbe7YWGlKCrm+vNWpun<-tc#00Y)4o>Zo~xRzS}c-^_9RbboSymO`{fJe zY3H7dyJDWdA76;4-Tq8j#r&!Ho1kDiKRLgAK3&<8E^D1RwdO0o?^tjw@=N zzwcb}fF4Y_S~AX}r1@9h>M&ocfVRy)v{ZKZ$W8kc$wWc!=)<=;_-qeWKTumAbV~v` z=!CxIfgIiYdN@pxJ#kN7c=;Q>6LGV4sdqxg{pvl!)-yU>nDJbUf8xSSP5dWlQ=lLK zFGA7&Me)NoK6oRw|M`{RL-HT3KeaxXeiTSKds3zzw)O8UHb?KeQk3s zZ(Si@v3bg6uYp&5hsyP@-;mEx5rA@b`w1|6R({+TW9xHXI6uMXE2(tcjIfH*6KF;; zybAMNPm~&Qt5QlFkA=8vSuSw9hH3);91_s76tLhgl0k$KaNp6iY@z9JlpPNt+D6l_ zjWeNaj*+I};v8iC5*m(jtA+4`dciS@sNX|@vM=94l!9H}LDZk2;LlO;DFn-^=Rw&> z{AHAi0>n^2_fcBMJSH4GXUC!#51L{<0DdLa5VU#WyToy?5ghX-*zl*8%kGtOnM8X8WC;6^zenxx*5?Hm9`?7VTTF=*$^H-h*_PJ; literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b47f6d78ac046b6b7b2fb07bd7f3382773ffb73 GIT binary patch literal 13340 zcmcgSYj7LKd3SgdPZ9uM5+$7^N+e`HtheXRL2vOwL7M#p0PYlTRUwhX_HO}Lk{7K+^S7xI{8PNwiD-1`+a-3 z0|-HOl1{r47km5o_Sp8VU^Pt{DBjDJy-cgv1Zy}iXTM|$)^a}1 z0l%vZ)KQ9aUZOY`?|#{s>$RHHx}ny?m+Q54q_!MtD|nAyyJpN7@UuH$=rhqsFT=4C z%M1yLkqE~NBGb!;&+{D9JH|)@JabxLIlir(8DfQJhIxsJvgi30yZ!V4FY*P0hN8p$ zOe8KP$RJT-;~Xn+41pix!vkzQBF6SIJv~^{5{XBo#xNV@u~$f&`anxdOOrhk7bRgh z#>b_V4NM=;O2Y!bqBhXe1GD0EQiPRw5oeDvFgep3iF3J66)Ag9Pdgih{;hz;aTt>; zcB)`I;PZmO4hN%W~pZtUsl zO27g;V6ezbPvnRTbMT`>&{K>g;tU_}kHq;VLU6!cf*BtjMN;+i0F^n<^FyNjLP9tn ziT5+ZakT|j#)sKqAU^I6P@EU~*f7t86LCpk!x9q_8Q4dl2Taf8_M;`1A*fr!$SQK*K&#%BD@IoKuUfzeZ@1E=cTL{Co+MuO!+8u}{$ zunfY-9VClC9f`q~u(2UP1PIF^&Kx@?w%C7#H!eUc4mKW7NGvX2><*X|Q%58$DW)!f zqS#IjVLcmF%xB_I2pE;}_S|{b4uj&!tzNJ2;;~fEzE%hGQ~*svu>?093+QXjg1!a4Rs*b6cl_@kE}fx=Zv(%WQ8V^f3JrzpLg z`{w8o>J$|)bO#KI5s0W5wsKg6#VY1#;sP(s(TWMt5fL);+s0P7h^>HGGh&7Xf?_M8 z*s7!0GBl=?g#aE|iBJnefumDm4P+OoyAF3!dT!5^Lz9Oxj=CAwf}?rC)VvIBRM>!d zF4^Fhtc`)WSFDZJ)+TgF`sxNv+Q9s;Om$FK3}Hh*Kt8vLq9zOl{Vn-^T1hr&kO#}O zy8N$3wQ0lU1`|M>(0iYv3gCVo9@W1O9xzvJ@mygLc{uw8$hA^{9zO;rY=jpiM!Q7} zo+@%xfb-}Q01g>&urWp)Lhi&8R!oG4CP`WXX0m09VTe=A$C1S-W%{jCOt9BHTpJ@y zBfwfHg&q}g7Z`>SaEBGQevd;iktl3{%0~DT`yjhW{nA=K<-hdoT~Ec8(aF&_eVLjA zZ~ErjZg_9jU8|d6udiFIZBExVzgydS+n266kntQ`upPYXa9w$1@{zf4#=mVYJX`VY zC$FBJK0CAih0ts`?cbJhY+o>K|MWv6ReSJL#lKB-z*IN49I@Em-dK0UVGF~jo^l&R zFE>He*WqaKRFDRp!obm>Lzp;olIjEP;*u+9;*6K5AbgL~K?__Ii{d9&xrkwmpFv@a zj7TTFz_T3FCnRFPwDzNDKG>?Ywd5o=a@{_VSSZ%_G9ck2Bf!eZm|j)@-Wt}=HsFa! z!#BO$0fiT6xDW8Frb(kreG^n9?R?Udlv9`}N~YmV=Wdlug>;Bx)Etex(JT&wU{!1( z;NKA`6cXG36ZW>Re-0R@$VdB>GDJTlBodNhiLt_YUV_UV3P)K{4249*3wHh@*?IxD zd3;SN8i5E=XV?Z^#5%}+1FXR2nzW{EHFs>Q?)uiGBC}N)-@{4MH*LZy=&)k_Iyd+Z ztfn+Km@wvVp{(-=(VM3RN6Yke180zp*D22U=VtA;{|PNqgC+REXK{fElWY?I^#SAX z_!TADsLeKEmd%5%qGtHi_D}dO96~Az%Ik!MrXtjYRq|=QVPvIN&i``yO0+pTsB=BQ zXKb=%5btPFl5L!&pMJ%vos5Yxxoi;GbW!3g1@LwOJUd6pR%x}?Z4mddD9QH0bwyS1 zsjW?eH@N=6efC4s@#My9J6Asz*IaoV1 z2nGG>@1Kt=d&REx9Eaw98grZ#*n^{c+>FBVz_^oX)J6J%@e2V|{y-;T;I<7V1PSi> z7%%N5nj+q=Jl{yfqhp{l0v{JatBJses4Au6K8EEuhGk-r_^`w`NBI#xnjZ)HWY2vJ zM!<_dUaqQN2exR8N1-LqUaa9nbT}3lfA!yx1Sj-_N?`6|V!$Vq!>1|)eBBA@cx))j zqvFqVj|xIU5CYJw_@ad>8;E;Ql@%+nt`HXy6bnBJ`j)78hQ?SJb3vcnnqz;8MI@9^ zO#RVBuVP1z8IsuED6d#l)C4?T!djfm%!(mYD&h$t#zrIKykbJpqRJ(Xk4kL7DLjPT ziA1vW4WmV(l1CtQ=U$?WF87k1s;Inbo3<^MH>Jy)GUY8v^Lrluy{c7L2c`!Wt2U>rHqVS@s8UJDzp#mp9#ec-xfmZtc1m>r2jC&V}Yf8RqcMYTKsTva9NDHeYL=rRUq{ zccz*%tM*K_XV)+{gV%zy#_xtwbhiCy%6Q|*&8}-*vlXva&sD!%o2lErSa%>@cOX-D z=$)Fw9~h~%%}Zu#)e-t58&$h5CEgspHhO(LQ?q5t{GP8STU|R<_8*Rld%l{KcY05< zBkQld@#M|3*Urw?XV$d7zTu6g*P7mHyR)q`y{2=a?r7S7H2G-O<4-yXzn?R7D~2&L}S_FkPLV+Tp&of2|n*luNoDJ}!ZWihwDvz`q;SU-vHCUC$NGk!(J4(06H0lHe`2MeH2=k!jQntQz&|5H0=Y3nayyBv%CU-oQN3EoUlm$hY08y{z$cc7j_{#e@9~BQ-K?UH z9Dozwg$xb;%Bj8^(u-$aIr8eUxnr;QzA^aP;KJIrJB}moHEqlIw$F9WJoD}L6#abX z)voET7rJLV)4uHs&h6^J@Eym78F9An)*0*AJKcJCcKuszKRo*O z(SJHV)pqsh^wH;!Up+Z}@?HO?h1SDqU-yEuJL{;JI*qns+Oa;{^zf@)b6vNq-f4pI zZRw`NE9YHkfuNYC z^fu8Vn+2lcMnKKw4B-1V#EcVVGzCh03Acr8dL=)DU2D=1 zN!z`3>yyTxSFK5QX8l!Q1gza~bKu&*;@a)$wcF=AZy*2J+RkbFlzB?LTUnD@_2QE+ zoxOE-ep6=CBbmxQQwFT8d2!oI`)}=^H)b~O$W-o}G5~5R@AHp<&4E&6X2VSHOwGIg zElXw8x})@mcB;HGyLRK0ajNrMj-?8!dhL>*s@`xlF`bx+WvX_(Q?>WDA?f(}n#Sab z_npnZtX`XXW~O=GmELeLQ+;T$`tfx2<3Bo)sSYNOz2{w(+CF2SJvPr}y!$5GVTQGV z51f?Kch^^)GCsd&!C9B}c$0QwECh_*LKo1xJ76JpjnEArd25D<8az!z zXFqo?U?P3T%SpJRphdB$QK9j&E;t6nMhcj^y9F@;MBHqaNb60RO2@4Dw>mdx-AI&y`7$%-W# z<$dU5Y+J6enj3!Y-DLKu_7Z{%!=IjV42NER8mfxzC9aIK7uZLRoPFc~`^Xfu@K(;r znJ<|%O)6;PT$}}3$~ZS?gqpQRgTEJ8jBAn9d=O{V5y`o0R>EUr ziQzZ~OdAg{BU?r;KvWH8VMqEyD_Xz-#Q?ZsAF~E~B}|A~LevjXBLUA=h|*~osdyFU z02Dmf&E0G_(@6YVFp}p21pwLjm_*`=d=x^XpPLY%Mp$*l5V|0KGL6hDkY3>;#zSR9 zd$k7&MfU0+;38jXr;;x;HA&K}R!d2%)gD4GSeOkD@DL2iA+CN4TVTA1m;xyZE!Mgc zXH+*=Z{nbbAp6Y&edd>K*`2$Srpun}o_$HvySCNj=9Q)oJc$4oI)(=N3=>As04Nlc z#c6~VdZ9%(Wb^HVIvw(Lph?i0?t$*mK{Sqwl6GZY?CU{osHdLFc^|C8H{eTEGHkecL##9)?8dq}$P_2>PHLXZvJOyHIx$?E z^Gkr0rfU!a(F=^=o)pj-BqFWIu~&t*6GFw{=)^g=zHz4Ht{3$99)jQ?yGUg{wW&P| zC~C6~_Y{YcEA7~nE%&Ql!u9F$^)rFlV5a<$WLtL6-oLMTv*!D&Ku8|Cb!dL;osBzx z;{9peKiBBt2Bup{%JP5NCo z&?r0y7^)knx@$Fl0Rj|njxdE8?u{@FnRW%4oGPX6igs0DXX+okGx=VBg^*CV-?ux9 zRe?18cLcHx54|*cYjm;UK)T_;?Z}T-Wg1Q=I~JYm(#~}=#hm;^bf1)1ago=Ta1hhyXIBekhRF&Bv6?qiQBqlswTavu!)BkILAfht@ zBE>Gxm8U14P8~^!*N$df^-06sHQ-FY5lRYIh9`%wj8Bd)*mlfsfJ`M?Az#A95u%+2 zQo_-t26H?G>h%OX(5Q1Qvuwz_?ZAC>GXEtLYk0yUgY8}Fp#k*XEt_Qv(xDXRlh6-9 zlE&Ev9a>HP7jSgtzqAs-)t!IZ=u^}faBt3V9yB^3$Qd7q)#IG$fo%t5GiOAM;n#$} z<3PXvKkW0)MxY)I>nZI6DBK|{NEzz`=?|^{C-k?LrhmXZ{-r0$!z@)lA!;AIr(hZN zeHOm5P~T=@=&2R&33-gcGiOyOY0vs>HJDS=qgS7ue)h%TOm*{O z^{#aFu6gN=@z=)R+W3}~+1-_?KCw`KBI~QpI?AVxz2iXhN*_PN&4yn+KX*R8c~7=x zb*lZEZD#w7bZbw#rfIQyN4k2)>t{06`xdJ^($yVro&I6)?cl-_rxwn9CG%)7Q~lIJ z`BN&~JN|~WqXEkIkRhYvZzecjoAH1)g|l{`QJHoy&>>}>-nZ!6l=f|UkYI}1-)@fj->qbDFR0vx%Jfk8VanSLez za=<9*YE}L{0q|y1H;MHM{V;%dV?gRfu{?lT470P4X|fRaKJT$8l7%jO>!&~A8DC$L zQx|smDL@mE@n59gw{5=9FW~V_wLdquV5?DyQ7ClqxDg)x6T+hle4@Jd3#2HwyIdyQ z&vyWM>+?aG!E#tgs;Nf|X(Hz9<#6GDS`B3G2oS0%^h(Z; zo_ypzl7pMnzJ>Yc!p~^|6RywRAGxr7bKwkXIlmLyE`XOkr?kO^GT zn5c28#9*)+9TzUbukgp19fk~Ei$bW$G{Qzj5&im__)vdUA!5y`%;T#|F_T;)28(hc ze1)zsjaDPQ|_d>z%z-mg-N@a8GpvT8Jvzz&y?}GV@dk1t9+^gVn~jRzdqyI z1ezjNOnqUx@`f>W{JJ~iYDgNg4%d|78@tgZxo3J0m_~bV?Y-k`S#Y-e(z7NN2A82Q z+jeVYVQtI2DdX7*jNk3O5}S z8{1RT^*t~4zSMWC@AV^ZJoegScYKE?JMQ_auI`)Omy$BR`bA%J+SjZ$F8G==zC+0l zdqi)coC3&N689I6>SN8g?mv|(h z$7xOHITvs}*da+28`M=4C!9feS03Y~fdr>2GUSls6%qahvyv{$B(`9NK*W8Jfy?p( zgVk*N#A`C|Uv`_#^~-LPxpB!!S-c+_4CbB7MkplN_-u-VHi(XhS5egh zQTQkp^?^Eh)1`8cdhCFXSwguvN?kXGu@&2X^;P>p;a}hr8Kx*fwq&Gf`U8WLHho-2 z(Hnn3x&EDM$xto7pu9h)eE&r~k*1z_pW6IulbJRwQ;>b&rtGU1R`1A^?YwANw$OCT xvdu*A_{hDHu3T9C&_@*fe$+)DpnVH#T0f%T_oKsxFVOU1I@S6yMG6F~{{cPb`GWue literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/config.py b/code/tac-qlib/tac_qlib/data/config.py new file mode 100644 index 0000000..53c9007 --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/config.py @@ -0,0 +1,202 @@ +"""TradeAC lake configuration helpers. + +The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-lake``): + + $TAC_LAKE_DIR/ + ├── market=US/ + │ └── timeframe=1d/ + │ └── symbol=AAPL.parquet # OHLCV bars: t, date, o, h, l, c, v, n, vw + ├── features/ # indicators, wide format, family tier + │ └── market=US/ + │ └── timeframe=1d/ + │ ├── family=ta/symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ └── family=sp/symbol=AAPL.parquet # t, sp_ou_*, sp_hmm_*, ... + ├── calendar.parquet # trading days per market + ├── coverage.parquet # per (market,timeframe,symbol) loaded windows + └── symbols.parquet # asset master +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict, List, Optional + +import pandas as pd + +#: qlib freq string (Freq.__str__) -> lake timeframe partition name +FREQ_TO_TIMEFRAME: Dict[str, str] = { + "day": "1d", + "1d": "1d", + "min": "1m", + "1min": "1m", + "5min": "5m", + "10min": "10m", + "15min": "15m", + "30min": "30m", + "hour": "1h", + "1hour": "1h", + "2hour": "2h", + "4hour": "4h", + "week": "1w", + "1week": "1w", + "month": "1M", + "1month": "1M", +} + +#: bar-field map: qlib field name (without the leading ``$``) -> lake bar column +BAR_FIELD_MAP: Dict[str, str] = { + "open": "o", + "high": "h", + "low": "l", + "close": "c", + "volume": "v", + "vwap": "vw", + "avg_amount": "vw", # amount / volume +} + +#: fields that qlib core/backtest queries but the lake does not store -> all-NaN +UNKNOWN_FIELD_NAMES = ("factor", "change", "trade_unit", "suspend_flag") + +#: columns in the parquet files that are not features +NON_FEATURE_COLUMNS = ("t", "date", "market", "timeframe", "symbol") + + +def timeframe_for_freq(freq: str) -> str: + """Map a qlib frequency (e.g. ``day``, ``1min``) to a lake timeframe (e.g. ``1d``).""" + f = str(freq).lower() + if f not in FREQ_TO_TIMEFRAME: + raise ValueError( + f"unsupported qlib freq {freq!r}; supported freqs: {sorted(set(FREQ_TO_TIMEFRAME))}" + ) + return FREQ_TO_TIMEFRAME[f] + + +def resolve_lake_root(lake_root: Optional[str] = None) -> Path: + """Resolve the lake root: explicit arg > ``TAC_LAKE_DIR`` (no fallback). + + ``TAC_LAKE_DIR`` is **mandatory** — there is deliberately no default + A missing/empty value raises so a + misconfigured environment never silently points at a wrong directory. + """ + if lake_root is None: + lake_root = os.environ.get("TAC_LAKE_DIR") + if not lake_root: + raise RuntimeError( + "TAC_LAKE_DIR is not set. Point it at the TradeAC lake root, e.g. " + "export TAC_LAKE_DIR=/home/data/lake (docker) or set an absolute " + "path in your local .env." + ) + return Path(str(lake_root)).expanduser().resolve() + + +class LakeConfig: + """Path helpers + cached readers for a (lake_root, market) combination.""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + self.lake_root: Path = resolve_lake_root(lake_root) + self.market: str = (market or "US").upper() + + # ---- paths -------------------------------------------------------------- + def bar_dir(self, timeframe: str) -> Path: + return self.lake_root / f"market={self.market}" / f"timeframe={timeframe}" + + def bar_path(self, timeframe: str, symbol: str) -> Path: + return self.bar_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def features_dir(self, timeframe: str) -> Path: + return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}" + + def features_path(self, timeframe: str, symbol: str) -> Path: + # Legacy flat path (no family tier). Prefer `load_features` which + # resolves the family=ta|sp partition layout. + return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def load_features(self, timeframe: str, symbol: str) -> pd.DataFrame: + """All feature columns for a symbol, merging the `family=ta` and + `family=sp` partitions by timestamp. Returns an empty frame when no + feature files exist (legacy flat layout falls back transparently).""" + sym = str(symbol).upper() + frames = [] + for family in ("ta", "sp"): + p = self.features_dir(timeframe) / f"family={family}" / f"symbol={sym}.parquet" + if p.exists(): + frames.append(pd.read_parquet(p)) + if not frames: + flat = self.features_dir(timeframe) / f"symbol={sym}.parquet" + if flat.exists(): + return pd.read_parquet(flat) + return pd.DataFrame() + if len(frames) == 1: + return frames[0] + merged = frames[0] + for extra in frames[1:]: + merged = merged.merge(extra, on="t", how="outer", suffixes=("", "_dup")) + for c in [c for c in merged.columns if c.endswith("_dup")]: + merged = merged.drop(columns=c) + return merged + + def calendar_path(self) -> Path: + return self.lake_root / "calendar.parquet" + + def symbols_path(self) -> Path: + return self.lake_root / "symbols.parquet" + + def coverage_path(self) -> Path: + return self.lake_root / "coverage.parquet" + + # ---- metadata readers ---------------------------------------------------- + def load_symbols(self) -> List[str]: + """All symbols known to the lake (from ``symbols.parquet``).""" + p = self.symbols_path() + if not p.exists(): + return [] + df = pd.read_parquet(p) + if "symbol" not in df.columns: + return [] + return sorted(df["symbol"].astype(str).str.upper().tolist()) + + def symbol_spans(self, symbol: str, timeframe: str) -> List[tuple]: + """Listing span(s) ``[(start_iso, end_iso)]`` for a symbol from coverage.parquet.""" + p = self.coverage_path() + if p.exists(): + try: + df = pd.read_parquet(p) + except Exception: # pragma: no cover - defensive + df = pd.DataFrame() + if len(df): + df = df[ + (df.get("market") == self.market) + & (df.get("timeframe") == timeframe) + & (df.get("symbol") == str(symbol).upper()) + ] + if len(df): + row = df.iloc[0] + first = pd.Timestamp(row["first_t"]).date() + last = pd.Timestamp(row["last_t"]).date() + return [(first.isoformat(), last.isoformat())] + # fallback: derive from the bar file itself + p = self.bar_path(timeframe, symbol) + if p.exists(): + import pyarrow.parquet as pq + + tbl = pq.read_table(p, columns=["t"]) + first = pd.Timestamp(tbl.column("t")[0].as_py()).date() + last = pd.Timestamp(tbl.column("t")[-1].as_py()).date() + return [(first.isoformat(), last.isoformat())] + return [("1970-01-01", "2099-12-31")] + + def load_calendar_dates(self) -> List[pd.Timestamp]: + """Trading days (midnight timestamps) for the market, from ``calendar.parquet``.""" + p = self.calendar_path() + if p.exists(): + df = pd.read_parquet(p) + if "date" in df.columns: + if "market" in df.columns: + df = df[df["market"] == self.market] + dates = pd.to_datetime(df["date"]).dt.normalize().sort_values().unique() + return [pd.Timestamp(x) for x in dates] + return [] + + def __repr__(self) -> str: # pragma: no cover + return f"LakeConfig(lake_root={self.lake_root}, market={self.market})" diff --git a/code/tac-qlib/tac_qlib/data/providers.py b/code/tac-qlib/tac_qlib/data/providers.py new file mode 100644 index 0000000..8d0644f --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/providers.py @@ -0,0 +1,230 @@ +"""qlib data providers backed by the TradeAC parquet lake. + +These providers plug into the standard qlib mechanism: ``qlib.init(calendar_provider=..., +instrument_provider=..., feature_provider=...)`` instantiates them and binds them to the +``Cal`` / ``Inst`` / ``FeatureD`` wrappers (see ``qlib.data.data.register_all_wrappers``). +The rest of qlib (``LocalDatasetProvider`` expression engine, backtest ``Exchange``) keeps +working unchanged because the interface contract is identical to the file-based providers: + +- ``feature()`` returns a ``pd.Series`` indexed by the **calendar position** range + ``[start_index, end_index]`` (matching ``FileFeatureStorage.__getitem__`` semantics). +- ``list_instruments()`` returns ``{symbol: [(start, end), ...]}``. +- ``load_calendar()`` returns a list of ``pd.Timestamp`` trading days. +""" + +from __future__ import annotations + +import bisect +from typing import Dict, List, Optional, Union + +import numpy as np +import pandas as pd + +from qlib.data.data import CalendarProvider, FeatureProvider, InstrumentProvider +from qlib.log import get_module_logger + +from .config import ( + BAR_FIELD_MAP, + LakeConfig, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, +) + +logger = get_module_logger("tac_qlib.data.providers") + + +def _day_freq(freq: str) -> bool: + return str(freq).lower() in ("day", "1d") + + +def _calendar_keys(cal: List[pd.Timestamp], freq: str) -> pd.Index: + """Convert calendar timestamps into the same key space as the lake parquet.""" + if _day_freq(freq): + return pd.Index([pd.Timestamp(x).date() for x in cal]) + return pd.Index([pd.Timestamp(x) for x in cal]) + + +class LakeCalendarProvider(CalendarProvider): + """Trading calendar read from ``/calendar.parquet`` (fallback: derived from bars).""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + + def load_calendar(self, freq, future): + timeframe = timeframe_for_freq(freq) + if not _day_freq(freq): + raise NotImplementedError( + f"freq={freq!r} (timeframe={timeframe}) is not supported yet: the lake calendar " + f"only covers daily sessions; add a minute-level calendar to `calendar.parquet`" + ) + + dates = self.cfg.load_calendar_dates() + if not dates: + # Fallback: derive the trading-day set from the persisted bar files. + bar_dir = self.cfg.bar_dir(timeframe) + if bar_dir.exists(): + import pyarrow.parquet as pq + + cal: Dict[pd.Timestamp, None] = {} + for p in sorted(bar_dir.glob("symbol=*.parquet")): + tbl = pq.read_table(p, columns=["t"]) + for v in tbl.column("t"): + cal[pd.Timestamp(v.as_py()).normalize()] = None + dates = sorted(cal.keys()) + if not dates: + return [] + + if future: + # append the next calendar day so that "today" is a valid trade date + last = dates[-1] + dates = dates + [pd.Timestamp(last) + pd.Timedelta(days=1)] + return dates + + +class LakeInstrumentProvider(InstrumentProvider): + """Instruments from ``/symbols.parquet`` with listing spans from ``coverage.parquet``.""" + + def __init__( + self, + lake_root: Optional[str] = None, + market: str = "US", + markets: Optional[Dict[str, list]] = None, + ): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + #: optional named pools, e.g. ``{"sp500": ["AAPL", "MSFT"], "etf": ["SPY"]}``. + #: ``all`` / any unregistered name resolves to every symbol in the lake. + self.markets: Dict[str, list] = markets or {} + + def _resolve_symbols(self, market: Union[str, list]) -> List[str]: + if isinstance(market, (list, tuple, pd.Index, np.ndarray)): + return [str(s).upper() for s in market] + if isinstance(market, str) and "," in market: + return [s.strip().upper() for s in market.split(",") if s.strip()] + if market in self.markets: + return [str(s).upper() for s in self.markets[market]] + return self.cfg.load_symbols() + + def list_instruments(self, instruments, start_time=None, end_time=None, freq="day", as_list=False): + market = instruments["market"] + timeframe = timeframe_for_freq(freq) + + symbols = self._resolve_symbols(market) + if not symbols: + if as_list: + return [] + return {} + + # clip listing spans to the queried window (mirror of LocalInstrumentProvider) + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + start_time = pd.Timestamp(start_time or cal[0]) + end_time = pd.Timestamp(end_time or cal[-1]) + + out: Dict[str, list] = {} + for symbol in symbols: + spans = [] + for begin, end in self.cfg.symbol_spans(symbol, timeframe): + lo = max(start_time, pd.Timestamp(begin)) + hi = min(end_time, pd.Timestamp(end)) + if lo <= hi: + spans.append((lo, hi)) + if spans: + out[symbol] = spans + + filter_pipe = instruments.get("filter_pipe") or [] + for filter_config in filter_pipe: + from qlib.data import filter as F # pylint: disable=C0415 + + filter_t = getattr(F, filter_config["filter_type"]).from_config(filter_config) + out = filter_t(out, start_time, end_time, freq) + + if as_list: + return list(out) + return out + + +class LakeFeatureProvider(FeatureProvider): + """Feature data from the lake parquet (OHLCV bars + pre-computed ta-lib features). + + Field routing: + - ``$open/$high/$low/$close/$volume/$vwap`` -> bar parquet columns + - ``$amount`` (= v*vw), ``$avg_amount`` (= vw) -> derived from bar parquet + - ``$factor/$change/...`` -> all-NaN (not stored) + - anything else -> a ta-lib column in the features parquet + """ + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + self._bar_cache: Dict[tuple, pd.DataFrame] = {} + self._feature_cache: Dict[tuple, pd.DataFrame] = {} + + # ------------------------------------------------------------------ caches + def _load_bar_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._bar_cache: + p = self.cfg.bar_path(timeframe, instrument) + self._bar_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + return self._bar_cache[key] + + def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._feature_cache: + self._feature_cache[key] = self.cfg.load_features(timeframe, instrument) + return self._feature_cache[key] + + @staticmethod + def _keys(df: pd.DataFrame, freq: str) -> pd.Index: + ts = pd.to_datetime(df["t"]) + return ts.dt.date if _day_freq(freq) else ts + + # ------------------------------------------------------------------ fields + def _extract(self, instrument: str, field: str, timeframe: str, freq: str) -> Optional[pd.Series]: + """Return the field as a Series keyed by date/timestamp (None if not present in the lake).""" + bar = self._load_bar_df(instrument, timeframe) + + if field in BAR_FIELD_MAP: + col = BAR_FIELD_MAP[field] + if col in bar.columns: + return bar[col].astype(float).set_axis(self._keys(bar, freq)) + return None + if field == "amount": + if "v" in bar.columns and "vw" in bar.columns: + return (bar["v"] * bar["vw"]).astype(float).set_axis(self._keys(bar, freq)) + return None + if field in UNKNOWN_FIELD_NAMES: + return None + + feat = self._load_feature_df(instrument, timeframe) + if field in feat.columns: + return feat[field].astype(float).set_axis(self._keys(feat, freq)) + return None + + # ------------------------------------------------------------------ api + def _get_calendar(self, freq: str) -> List[pd.Timestamp]: + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + return list(cal) + + def feature(self, instrument, field, start_index, end_index, freq): + field = str(field)[1:] + timeframe = timeframe_for_freq(freq) + + cal = self._get_calendar(freq) + n = len(cal) + lo = max(0, int(start_index)) + hi = min(n - 1, int(end_index)) + if lo > hi: + return pd.Series(dtype=np.float32) + + keys = _calendar_keys(cal[lo : hi + 1], freq) + ser = self._extract(str(instrument).upper(), field, timeframe, freq) + if ser is None: + vals = np.full(len(keys), np.nan, dtype=np.float64) + else: + vals = ser.reindex(keys).to_numpy(dtype=np.float64) + return pd.Series(vals, index=pd.RangeIndex(lo, hi + 1))