From c2edfe6671f17a364a27ecf9619df25f194f43e6 Mon Sep 17 00:00:00 2001 From: zhaoli Date: Wed, 19 Aug 2026 14:42:10 +0000 Subject: [PATCH] start experiment 32 (exp/32-scheduled-daily-retrain-for-round-target) --- code/MANIFEST.txt | 27 ++ code/tac-qlib/tac_qlib/contrib/__init__.py | 11 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 374 bytes .../tac_qlib/contrib/data/__init__.py | 3 + .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 216 bytes .../data/__pycache__/handler.cpython-312.pyc | Bin 0 -> 11073 bytes .../tac-qlib/tac_qlib/contrib/data/handler.py | 254 ++++++++++++++++++ .../tac_qlib/contrib/model/__init__.py | 4 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 319 bytes .../__pycache__/rank_ensemble.cpython-312.pyc | Bin 0 -> 9461 bytes .../__pycache__/rank_gbdt.cpython-312.pyc | Bin 0 -> 12288 bytes .../tac_qlib/contrib/model/rank_ensemble.py | 189 +++++++++++++ .../tac_qlib/contrib/model/rank_gbdt.py | 238 ++++++++++++++++ .../tac_qlib/contrib/strategy/__init__.py | 3 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 234 bytes .../__pycache__/optimal_stop.cpython-312.pyc | Bin 0 -> 10428 bytes .../tac_qlib/contrib/strategy/optimal_stop.py | 217 +++++++++++++++ code/tac-qlib/tac_qlib/data/__init__.py | 25 ++ .../data/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 522 bytes .../data/__pycache__/config.cpython-312.pyc | Bin 0 -> 11464 bytes .../__pycache__/providers.cpython-312.pyc | Bin 0 -> 13340 bytes code/tac-qlib/tac_qlib/data/config.py | 202 ++++++++++++++ code/tac-qlib/tac_qlib/data/providers.py | 230 ++++++++++++++++ 23 files changed, 1403 insertions(+) create mode 100644 code/MANIFEST.txt create mode 100644 code/tac-qlib/tac_qlib/contrib/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/data/handler.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py create mode 100644 code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__init__.py create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py create mode 100644 code/tac-qlib/tac_qlib/data/__init__.py create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc create mode 100644 code/tac-qlib/tac_qlib/data/config.py create mode 100644 code/tac-qlib/tac_qlib/data/providers.py diff --git a/code/MANIFEST.txt b/code/MANIFEST.txt new file mode 100644 index 0000000..fe05cf3 --- /dev/null +++ b/code/MANIFEST.txt @@ -0,0 +1,27 @@ +# TradeAC custom-qlib-code snapshot (auto-generated) +# parent repo HEAD : fd5382caa4c4e96417a16d5a22006d9a7d625d00 +# tac-qlib/tac_qlib/contrib +# tac-qlib/tac_qlib/data +# per-file hashes (git hash-object): + 1b6298c4a5652f2e863cbdc385a1014a570fcd59 tac-qlib/tac_qlib/contrib/__init__.py + b419ee55ed455a1c45423d1c9025ca5cc0a98576 tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc + c76a9f17f680e74eea766eff27f7624359749ed6 tac-qlib/tac_qlib/contrib/data/__init__.py + 2f6c67620aa2f9e6aaaef3369361d9b3eac3d6ca tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc + fdd5923a70a399e8680913593ff111641947898e tac-qlib/tac_qlib/contrib/data/__pycache__/handler.cpython-312.pyc + 0dd25ef161c6e0f15eafc84886e7e1381deb38c3 tac-qlib/tac_qlib/contrib/data/handler.py + b151d139a0dcde87d74b21e7c4b729176ba5c39b tac-qlib/tac_qlib/contrib/model/__init__.py + 08dec87ccdf6bb5d2cf611ca3032a4280aaab8cf tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc + 6fb61946ea9a83dfb560de3717f5fbf482c4c00e tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc + 3e80f2e08b661ddd2f58ffe5a6196063fa41ae51 tac-qlib/tac_qlib/contrib/model/__pycache__/rank_gbdt.cpython-312.pyc + d3f051f3a8650c42fedc7b367b966f7c74fb5789 tac-qlib/tac_qlib/contrib/model/rank_ensemble.py + d03e6611338918d4aac5eea4adf26f85a3763652 tac-qlib/tac_qlib/contrib/model/rank_gbdt.py + 4afcf9058231111c412925f4c4b84e81d656db87 tac-qlib/tac_qlib/contrib/strategy/__init__.py + 6ad10c2ebe37c16417e67c7aeb731ad1fcb6da2f tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc + 8d684b3216b040071d9ee4fa920a0e0c7486d278 tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc + 79aaad9e39fcc740a773f4f63c512ce1086cfde0 tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py + 92e6e90eb0cd0a25142034560f27adb6b705b1a8 tac-qlib/tac_qlib/data/__init__.py + 7c4e6c345fad1978efe8860c0d977d0c02d6f8d9 tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc + 99e602392d51663cb06d5c425000b1ed1e5a916b tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc + 020dcdcf288e4832c8cf2386351f78d5ceb4fe13 tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc + 53c9007a928841fd3c3b08450f9a6520ce1ac091 tac-qlib/tac_qlib/data/config.py + 8d0644f6f0d1efb94798ed444cc73e63b643459b tac-qlib/tac_qlib/data/providers.py diff --git a/code/tac-qlib/tac_qlib/contrib/__init__.py b/code/tac-qlib/tac_qlib/contrib/__init__.py new file mode 100644 index 0000000..1b6298c --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/__init__.py @@ -0,0 +1,11 @@ +from . import data # noqa: F401 (registers tac_qlib.contrib.data) +from . import model, strategy # noqa: F401 +from .data import TACHandler # noqa: F401 +from .model import RankICLGBModel # noqa: F401 +from .strategy import OptimalStopControl # noqa: F401 + +__all__ = [ + "TACHandler", + "RankICLGBModel", + "OptimalStopControl", +] diff --git a/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b419ee55ed455a1c45423d1c9025ca5cc0a98576 GIT binary patch literal 374 zcmY+9y-ve06os#U@}ojKbOi=Bf&_g5gajP`senY8yjZ4*s(_sY*9;vQI`9MxY-~IW zZxAVBU}8gbW8yl3fjb<1=bmd{`?cHc0M^6P;(AN~-o5!N`U}eu!Xu=R5)Fn3LOfGs zDyU(CwVr8WxZo{wO-&$zmIKX9C_(~LNW*6$qIs0WAGiw=Qu9TUgS>7-D+OPd-(1< mo$cEm9`%2kt)YA}LP!JsH`r)ky@AaJw!S!Dq4$wp#ry}E*;*?A literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/__init__.py b/code/tac-qlib/tac_qlib/contrib/data/__init__.py new file mode 100644 index 0000000..c76a9f1 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/__init__.py @@ -0,0 +1,3 @@ +from .handler import TACHandler + +__all__ = ["TACHandler"] diff --git a/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f6c67620aa2f9e6aaaef3369361d9b3eac3d6ca GIT binary patch literal 216 zcmX@j%ge<81f8>6v#fyhV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##>w=j?Ny5c_}%mMMca&AwNy#TkIJ~0=L-X;}dgo;^S8`d7ehzB#0^YcoIfC4FrC5ig+@tJvui+AWXvKO%f#Q0OQZxA2=Q6xoDTD=LG57U+-S+XTl5*1sfVM(O?fa8EG>;eRZ2fYi5 zA`3Qh>Lie()Kb!n=~%AS#50xaOsdZ0hdl92ZBJ+FOq(eX(j~Hyr*576@ITs=r*Wn~ z`klL20F>!uCEmSzALqW#<2&c#-(9WBb}{cm9}I__&xKm-6E2sZ?6lL>V=jlGnt9n#-U@5l2-?Rt67?Y3Yp8=XQ^dr!*1& zscK7nO3R~J1Gz1tlFh3#;$$wB&5PZSPUcg&tavEAk98O~=0W{cC84NFHm=CEq1@9s z^-Ln2n-*uJOnRp#j*V5P(^u^{HYS1s`bwDUqYn-reoVYWoK%(GcrG(Jl?Sc7)JqFY zC{lh(RWwn{t5PaCo_E*QL+cPHCH0vpB`+pYX*9}@tGTJX(~?x#*j-e{6-~>j;$&)4NvE=k*gZCu%3{H>TAc>s z{TMN=NNRRfaTw%M>9nX$Wi>Z8FQ1$25w)DCoJ*l1o5b?tpo_Il#m7a7w&{qPo6JhX z(rG0cUFqN^fleyD4uEuir8AX{CY1fJDR5p%nPAYPr`(t9F zFt2;0Y&Msd=nS<8zI{n0pO{LgB{iu{D)F$O3&WU{?ljI>O6#`KsmZjW+fHTC2R&T1 z1>>Wsfl-E!qaXx&vA6_+r_)O8%rw#cn7BP`)!oB5sKdEzB9+vG`BX+ps8U9WC30#k zp(@Ylp~$0=*wLZE(NiafVuv3cKJ`dsM0e9p45rf&De}5Sx5d+vrs;NruDS=+YUq>{yJ9E>(A zYy$(&+PB5iIZZ*~*=cEV*0JquEIONaPL68D(bZV94R|| z^DEzG$}Y*xSqt0|?&&uR6M~tmzu&0VY8Uekv$U~~S^qV(o3qW? z^R8Me+Z)wUCYVY#%Z>M(L*@&Pvz)qBwy^i60{^U~U^(wN#>v(sVY-X^~J}P_?Z}V zPgVoNW|(|K4!cv?2_;TyCofK?vNBX1)MSp-Nn8u}={{7%piI%OWjUu#z^oo&OR_O~{)J`m4CG(&Lkm#OFPM(6)O-lLkuv>Sh zv{aV#SX|MYYeE|{8tJa8=;Tz@LllV3bb-WGw@*!C>5A@*#j+%zF

T@w{%ACLtDb z*rnTvTr6FmfNskw)3LP9LyFEY5p_%&GOBK2Q)3;lv5()96wGT9V@u&R?$$9QWukZZOZ<|6z{-(cue)D2i?^4&E za@U?k|K6KjcU})H?mn{6HFP0T9J;jUW@z)(?F*sqrBH7<)LT4K@o&1iyX@b2EpkIH z_Z+IUZZEYCTu+z25?SUffna6bi4`l}rPj!@oeQ+R=)3H@-d8&L_420JV)IyKTlnuI ze;2utEN?hke3*@XWpvTM2l$&=|ynz>nFAUFjyzkxq zVKpE4-lNN1oNwLoM$Q{JKfHnkajUe54Fyb#kOY0l^&;0GNsvNQf}mTh6OG!32;nKz zBy>(F2opr@w#Sro&64Cs3^-=Z2+z9!m7GOa0+8%E@0=}9e8V2I{hVDEpv`tb>o_2_ zwu0kj?iI@{+RZr&&b4%g*;C^$b{6awXpFiX&#ps1`^9d^DKr34`IB=Ep_Sv$wNkw< zF7G$zU9g+_V1%oVUegf9=xetPyQDX<^+!@#Ja<-6O>82@$s}Yo!Y>*?6EuOiOF@H~ zU9~0zPe|1e*s9_$l7z>l^ce$zjU}W^Dm`<5UMgslV+3pRDTcVx(oAj&0BkZnr2#X| zh-oD$#b?As8d#5^o^YR=XpIgsNHuc2HXqU(a!Q+E2=4R@%Apt3sW{PPK!j!kz?A_u z0aOH-V}xoDY?9WhP8&hA#>Tcm2V=VvsdM2z zaVRsHpAl0DC`og}4dNJ61F9h)L_4IC{fxwG_7-1cnPFA<6#=C`xzuH@N=@xYhzLtI5>`VWMWGP0nkS!U<=+)XbUm>QwSIdu`Uuq69L|pSKjOSpnA9 zI*ZxN@daMxFCO8!EDy=y&+!^uqdFZ^;zqcwT;6Mrw~bS+JohAbacr79XMK{J=EIiA zY_oB!_wVYfqU*3jw;Si83(0iuv~C5v=pI7vF%5b-Bf${RQK=+jx@+iMTw(q~*rq#h z=rK8^>UM+E)G##>z)zZPnS4g0{i4G;uaX|ueKCVcVn*xP0R}7?kH)H)J!YV{zWdTS zAWrQ--|8+WKt}s3B!4*1RRS$V`^`Y>yyI2xOWxAy52E(cSek<>FdT2~FG z>(+Q8sXJ*H)`yUodM^!a#f;hX)cyF>h)?Mp+4F#LnC)MBc4qiC8=;^HZ14Y*dg7}%>wr#rA#)a1Z zVEg%_ANf0q)=F^0{Pb1rsxlud2YZT+iYNHOfeQ!T_jFXe!OPjJk1wq6Eqi+}99?#B z&7I3`&K0<{zwF$&{xVkUwh()r`&sZspsKx&%=v7k5ruATXxPHytL!H2TDTw zvXyiBFZW$-TL|tdJ9mMxJb`MJ+K*Ml@mG6D)!^Y*U=7mX5!0~4kn&NO zEm^?ZCfg82u+La`1+bIya3)Q)-mUZ{`$Wm~x}j*{g+tduolnCt#bAtV&G%-| z7OHoIJXe!Q985PuN|?FAR|+PD1cX9^Uuk}?qjQF&7qzcU4$oKB4AF)`(CwAD2cxe29xYZHlLn>_bxJ{&IoHtGDD^w-aN%6vi)bJbV}~e zD_UM;zBpnE**rEG#?g>abC(QL6p1$?G-4Kf^D6{rUnGx;iy>z`1q;5a_J-q<9KNc| z#c?y->wF|^Z2;t6QmnCjE=KzR=UdQ}bfR9b(XikTLq37@Q&~7BvDpwYoZ)WwnT1nm zs%=05J-V@{C@eZVDxKZ*Zm)E1EDG;C+YP?2O$TksS%rfXAMo0bWiW$RA`GZHsv zX_#{jHm{nqWi1!?2={^z2Y8NK7#J3CbOO1s6J!Q%drX78mQ!oi0UmEJ2$wAchT|5% zb}m+f1GICunR3KPX)hbic^t7+>N(AYZ4ril5UdF#lzefnz$ zE1}ln!<9g&Xs>wuOP-Bo&&HBx*R}jo{{!Xz2TJ|J%N9pd=LgM0|J}Z;vTpqq+q}HA zaiF|$;QH>RyN{IbKC;j@^j`Fq)v_UY$$rbtdHuz+-`zjI`PZ-=jC6P3Kl=W`x6paw zb>G#28#~?(ycz!IzBl_8_8nW=7b)+Hyua`Gk4{|kl{-%u^5G8^J#|3PS!wV1zMUXJ z=l^}`pyfS_s;$uhk4=MEdr%?dJY}1zI#89G~Or=Bm+#UoB&ZzU(Fy^6_7x!G=b9ML9*4^c;yKg*kY0pCFXvuq& zS$U>hfJYe$k*;BU=2*_tEJ)x0`f}Y{fr8 z5fmHc6#H80fz>#X+7iAZ2Y=fB94;{wx9pJ}&k3f`M?EO<%Fc!ouj~RuXhL}t+Pa^! z)!Ks2IX>#g;rVoT{eD1-5Qcz|K4`@0aW`;Uik~qYH!_6gU4R)rZ}3TvOB9~X!*vof z0EsN(JeZywmj>?HJ2titwvxh2DuSTdJYwJdV`KF=7yGOQuBeV8yztdzxN2F2nNAuW zA$ye!tf2$;ll?WUWEI<^2kaICY=(1ef?U{ZB_Y9g+$XBibnV^%Ob6>qZmcnYadTjz zl0-#;K{Tai*$ehnHFk(?wUyD< zNDH2VP-rUn-XN8LXUIDpP0R6MOE&!$Sp*5dH*I{Hg@KJYfhS{FKEC1=~s z+Y3Hy?^UaItDY@vOu==+Vo>t?WX*}@0*tyrxg-!f-y^1<)hx^rpE zp7NGGH*5=A4wRjnitbAf!_Qy81Igz8>%P*agO!cl*X}5FedQCIwIjIf+9-CSwuqD)sW{cqp?i61ox68KwI0>~LQ zr7E{rasycQ!j6(zKYz4AA3(nHH4T9HZdGb)x@iyv5RrYsRt2Lt$~hbE zcx-g|^|ohu08GDG4w%`(U?yud$7LU>%YA`sh(lF@iF!Zu3k4x=d*m8mZkWMVngQ;G z8ntG%pJ~f5mvw)-1!lNi;O2-DZqI6~gK#t2Yk=&!HiWBT%L)x^?F1z|^e?eFd(3&@ zcFqaSZAVNGaD^!2PKSX}Y2fo*aC1|w!0PK815Nz}D9mna6zeb3 zXYZq>_8Ah~(RkV+cW5r0R>xthSd}JXIdGjxC zY`*l&i-pUDtA`g`yDx9P7Qa6Dda@k6=f+j6LbExS4{L>ch-f8~x|LJ<-b3G8z zNd3Oie0Y!jEeDc!dRh+OC%kJ3Qo3Q&5nhPbPyiZ|EJN=3mFH2q`j)4DKWsvG6*wWu z%Y2egSVmUe2<2;g-81*H<&~NS`zCrbIpyI%JK!D-4;~sCzVl;Jfy|r!m=12%!HT!bvl|eGnwp$6 zt}vLnq!URFTk*HI|6*cs??VgX zvGU%Bt`A&ozjop$U8TJbl|@YFSjl_rrmwZMVfXb@*B>gi9a!)kEIAK;{;8GQ{4lSP zD7?GHdziDYk`UT-CLyF2At6YOXM{6+Mf!!x(9ywD!=o{FG+~~BB#k;&0}Ra+n>G-Q zaYU=O_4dkTbkyj=<~(=P-&t~Y8pgb4o>5bad>KK!MiD}6s@Af84Qm>5bJCW&sz*rg zC`cMWjG)}=xLqr5_1G2UZK!c}O9N+oNt+d-Mn(z@;_trJe*Ng;eUbO_3-^sK_)nFb zr`W^Gw7zKk%XI>zpf^zAW6wM z61~klmxm%phfcR#Y; z#8~01j^|{Rj1iL$^<~PvLYKxaZ|ef&vxdMGbTxyQyLbhFuS;uM*j-lzfL0 zcAg|N2K#TO99u1|TSrib#gv%eWNb?>l{@b7_NHQ{By?7Mo!2(rXf1UQmW4yhHmgzX<<>{|5*PfidEk80r}iKpxV38s zzv0$Nej6Vw1vW1Q?kETDxW(b?R)ml9{A2v3u3H?-d^&99JV*HRj%6>mDO}>3-{(5m GO8+0at1T=5 literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/data/handler.py b/code/tac-qlib/tac_qlib/contrib/data/handler.py new file mode 100644 index 0000000..0dd25ef --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/data/handler.py @@ -0,0 +1,254 @@ +"""TACHandler: a qlib DataHandlerLP that builds datasets from the TradeAC lake. + +This is the "custom DataHandler" entry point (Option B): the handler is referenced from the +workflow yaml's ``dataset.handler`` and reads OHLCV + pre-computed ta-lib features straight +from the lake parquet files through ``QLibDataLoader`` + the tac_qlib feature provider. + +The standard qlib processor pipeline (``infer_processors`` / ``learn_processors``) still runs +on top, so existing recipes such as ``DropnaLabel``, ``CSZScoreNorm`` or ``RobustZScoreNorm`` +keep working unchanged. +""" + +from __future__ import annotations + +import os +from inspect import getfullargspec +from typing import List, Optional, Tuple, Union + +from qlib.data.dataset import processor as processor_module +from qlib.data.dataset.handler import DataHandlerLP +from qlib.utils import get_callable_kwargs + +from ...data.config import ( + LakeConfig, + timeframe_for_freq, + NON_FEATURE_COLUMNS, +) + +DEFAULT_INFER_PROCESSORS = [ + {"class": "DropAllNaN", "kwargs": {}}, + {"class": "ProcessInf", "kwargs": {}}, + {"class": "ZScoreNorm", "kwargs": {}}, + {"class": "Fillna", "kwargs": {}}, +] +DEFAULT_LEARN_PROCESSORS = [ + {"class": "DropnaLabel"}, + {"class": "CSZScoreNorm", "kwargs": {"fields_group": "label"}}, +] + +#: always include raw OHLCV; ta-lib columns are discovered from the lake and appended. +RAW_FEATURE_FIELDS = ("$open", "$high", "$low", "$close", "$vwap", "$volume") + +DEFAULT_LABEL = "Ref($close,-2)/Ref($close,-1)-1" + + +def check_transform_proc(proc_l, fit_start_time, fit_end_time): + """Port of ``qlib.contrib.data.handler.check_transform_proc`` (inject fit window into procs).""" + new_l = [] + for p in proc_l: + if not isinstance(p, processor_module.Processor): + klass, pkwargs = get_callable_kwargs(p, processor_module) + args = getfullargspec(klass).args + if "fit_start_time" in args and "fit_end_time" in args: + assert fit_start_time is not None and fit_end_time is not None, ( + "Make sure `fit_start_time` and `fit_end_time` are not None." + ) + pkwargs.update({"fit_start_time": fit_start_time, "fit_end_time": fit_end_time}) + proc_config = {"class": klass.__name__, "kwargs": pkwargs} + if isinstance(p, dict) and "module_path" in p: + proc_config["module_path"] = p["module_path"] + new_l.append(proc_config) + else: + new_l.append(p) + return new_l + + +def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]: + """Discover feature columns present in *every* feature file of the lake. + + Walks the `family=ta|sp` partition layout (plus any legacy flat files). + TA and SP columns are disjoint by construction, so the common set is + computed per family (columns shared by all symbol files of that family), + then the per-family results are unioned. Returns sorted field names + (without the ``$`` prefix). Empty if no features are persisted. + """ + cfg = LakeConfig(lake_root, market) + feat_dir = cfg.features_dir(timeframe) + if not feat_dir.exists(): + return [] + import pyarrow.parquet as pq + + def _family_common(fam_dir: Path) -> set: + common = None + for p in sorted(fam_dir.glob("symbol=*.parquet")): + try: + cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) + except Exception: # pragma: no cover - skip unreadable files + continue + common = cols if common is None else (common & cols) + if not common: + break + return common or set() + + common: set = set() + # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet + for fam in ("ta", "sp"): + fam_dir = feat_dir / f"family={fam}" + if fam_dir.is_dir(): + common |= _family_common(fam_dir) + # legacy flat: features/market=*/timeframe=*/symbol=*.parquet + if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists(): + pass # family layout already covered + else: + common |= _family_common(feat_dir) + return sorted(common) + + +class DropAllNaN(processor_module.Processor): + """Drop feature columns that are all-NaN over the fit window. + + The lake can hold fully-empty indicator columns (e.g. a ta-lib output that was NaN + from the start). Such columns carry no learnable signal and make ``ZScoreNorm.fit`` + warn on empty slices, so we drop them before any other processor runs. The drop set + is fixed on the fit window once (during ``fit``), then applied consistently to every + segment so train/valid/test keep identical feature columns. + """ + + def __init__(self, fit_start_time=None, fit_end_time=None): + self.fit_start_time = fit_start_time + self.fit_end_time = fit_end_time + self.cols_to_drop = [] + + def fit(self, df=None): + if df is None or len(df) == 0: + return self + window = df + if self.fit_start_time is not None and self.fit_end_time is not None: + try: + from qlib.data.dataset.utils import fetch_df_by_index + + window = fetch_df_by_index( + df, slice(self.fit_start_time, self.fit_end_time), level="datetime" + ) + except Exception: # pragma: no cover - defensive + window = df + if len(window) == 0: + return self + self.cols_to_drop = [c for c in window.columns if window[c].isna().all()] + return self + + def __call__(self, df): + if self.cols_to_drop: + return df.drop(columns=self.cols_to_drop, errors="ignore") + return df + + +class TACHandler(DataHandlerLP): + """DataHandlerLP backed by the TradeAC parquet lake. + + Parameters mirror ``Alpha158``: ``instruments``/``start_time``/``end_time``/``freq`` define + the queried window; ``feature_fields`` selects the features (default: raw OHLCV + all common + ta-lib columns found in the lake); ``label`` is a qlib expression for the target. + """ + + def __init__( + self, + instruments="all", + start_time=None, + end_time=None, + freq="day", + infer_processors=DEFAULT_INFER_PROCESSORS, + learn_processors=DEFAULT_LEARN_PROCESSORS, + fit_start_time=None, + fit_end_time=None, + process_type=DataHandlerLP.PTYPE_A, + filter_pipe=None, + feature_fields=None, + label=DEFAULT_LABEL, + lake_root=None, + market="US", + **kwargs, + ): + # default the processor fit window to the queried window (like Alpha158 without a split) + if fit_start_time is None: + fit_start_time = start_time + if fit_end_time is None: + fit_end_time = end_time + + infer_processors = check_transform_proc(infer_processors, fit_start_time, fit_end_time) + learn_processors = check_transform_proc(learn_processors, fit_start_time, fit_end_time) + + feature_fields = self._normalize_feature_fields(feature_fields, freq, lake_root, market) + if not feature_fields: + raise ValueError( + "no feature fields available for the lake; set `feature_fields` explicitly " + "(e.g. ['$close', '$rsi_14', '$sma_20'])" + ) + + label_expr, label_names = self._normalize_label(label) + + data_loader = { + "class": "QlibDataLoader", + "kwargs": { + "config": { + "feature": (feature_fields, feature_fields), + "label": (label_expr, label_names), + }, + "filter_pipe": filter_pipe, + "freq": freq, + }, + } + super().__init__( + instruments=instruments, + start_time=start_time, + end_time=end_time, + data_loader=data_loader, + infer_processors=infer_processors, + learn_processors=learn_processors, + process_type=process_type, + **kwargs, + ) + + # ------------------------------------------------------------------ config + @staticmethod + def _normalize_feature_fields(feature_fields, freq, lake_root, market) -> List[str]: + if feature_fields is None: + common = get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + feature_fields = list(RAW_FEATURE_FIELDS) + ["$" + f for f in common if "$" + f not in RAW_FEATURE_FIELDS] + elif isinstance(feature_fields, str): + feature_fields = [f.strip() for f in feature_fields.split(",") if f.strip()] + fields = [f if f.startswith("$") else "$" + f for f in feature_fields] + # de-dup while preserving order + seen, out = set(), [] + for f in fields: + if f not in seen: + seen.add(f) + out.append(f) + return out + + @staticmethod + def _normalize_label(label) -> Tuple[List[str], List[str]]: + if isinstance(label, str): + return [label], ["LABEL0"] + if isinstance(label, (list, tuple)): + if len(label) == 2 and isinstance(label[0], str): + return [label[0]], list(label[1]) if isinstance(label[1], (list, tuple)) else [label[1]] + return list(label), ["LABEL%d" % i for i in range(len(label))] + raise TypeError(f"unsupported label config: {label!r}") + + # ------------------------------------------------------------------ utils + def get_label_config(self): + return DEFAULT_LABEL + + @staticmethod + def discover_feature_fields(lake_root=None, market="US", freq="day") -> List[str]: + return get_common_feature_fields(lake_root, market, timeframe_for_freq(freq)) + + +__all__ = ["TACHandler", "DropAllNaN", "get_common_feature_fields"] + + +# Make `DropAllNaN` resolvable by bare name from processor configs (e.g. the default +# ``infer_processors`` and workflow yamls that reference it without a ``module_path``), +# mirroring how qlib registers its own processors in ``qlib.data.dataset.processor``. +processor_module.DropAllNaN = DropAllNaN diff --git a/code/tac-qlib/tac_qlib/contrib/model/__init__.py b/code/tac-qlib/tac_qlib/contrib/model/__init__.py new file mode 100644 index 0000000..b151d13 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/__init__.py @@ -0,0 +1,4 @@ +from .rank_ensemble import RankICEnsembleLGBModel # noqa: F401 +from .rank_gbdt import RankICLGBModel, rankic_feval # noqa: F401 + +__all__ = ["RankICLGBModel", "rankic_feval", "RankICEnsembleLGBModel"] diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08dec87ccdf6bb5d2cf611ca3032a4280aaab8cf GIT binary patch literal 319 zcmX@j%ge<81f8>6vrYl&#~=<2FhLogHGqui3@HpLj5!Rsj8Tk?AU0DDQ!aB9b1q91 z3nN21LkjaE#wgZG7ERWdj6mg@jJL#s67#Y>on7;aQ*)DYQhnT=eDhOMb2OQ5@j+x@ zBDZ*ofGRVSqzEXG3v!Zve0*kJW=VX! uUP0w84jZ6YX-=wL5f9J+kY9=gfW!x8Mn=ZF42Jg^j305iHL@3h!VLg6P*gDh literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/model/__pycache__/rank_ensemble.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6fb61946ea9a83dfb560de3717f5fbf482c4c00e GIT binary patch literal 9461 zcmb_idvFv-dY{?ZmsY!ag2cnX0K(D=Y4rdBmJOVESlGO5gYB|xZZ+DOl~&B|ikVpp zDXTlqMT%XHJ4BsbEWsD!N-8G@R~@O!1Lq(6sxEa`mHZ(g9Jc1TiYvZL#s3M*7yGVK zmHfV*on1)?DoJGoJw4OiUw41cUw^az7zp?{T-ob?JiL27$9+x@_T{oF{QNmou5vQh z$H~0xQ2Dr{&%sk&P=!7Lccv1?SM^n)UN*@0RXey|PWGM0lTvk4uvXAlGbRMfQ(MY7?u4Sqtx7^y;sdH8 zi$`LELq_kuqoOe+8KNR->R798Bu8{HnGhpdQrBB`C1S*q2}u=Sk`kw%+aL1z`unZw z!2+ElppcXmwZC5+)g%nIJ1l5u8d0=XSsE))5u?hRAh{K^;)`NNmOM zXiU?Mu+e!KJ@2J}cBBGueXpPf?#F#D`niNZ* z3+A`_j>e23F(xYsBNo9DhQ?@RBa$XzIz`i4L@6QrMl?l^v5o7Zq$#3=$)rJ$kr-DP zB|Rjb9KsAV$&eyiGe%TNXV2J~Aw^m>xm5$-iac+P=}A>G6d60xl&GR9iHIU+6flwrw0OtlmY~m2j61Mr! z-yejWPGVK|8xS5+G$KrEwdA)z;KWjKKpB$Wj3rZ=n2fS+I^L=ZR6vn^*7lkpwU}&= zm$lfNSTj+y<$|??Q=~CLadarADpo5A$>JeM`gB7Yh^evh!q^hCMQISy(hbO9OtfZa ztZvkou9bK}TO{ey+93pzp{goGR|cDCQ8aB)8?aF74NgF9j3G^tpcToaD*o%aw;?Tc za3I7wKr*1Vf9qD#1PDo0B)zcG-sg_Ahmw-*vNJ4A5apzpNE+gRnv9%=?f7VR zbQYVs3LU<)y*(aFv~`hT8KgA>$+N+bcoei#&^3B3i91?HhQ@SK%hpH|TGu9abc!dA zp7aeRb)&Q;UDVM<5x>)UwuMcrLc_^+X(w#s&?if8<4QcKjkOM7C!;YLia(G{LA_~X ziWC_V_a#9YmJuPdwX}oyg24vNkQSkfZEZ{$VH@BKHs7^i`@R8-fDR5M8VzQYV7MNE zVk%FIs-z9Vy4YrD5BACj6a%x8{U{>ovM5W2uxC`#;*2e@4#qMCYgExX zXp62X8dFtE`G&9`mF*%jqy#MsX4@kp!h1*~AwtWbA=nRXv$;7rnrLpu9!cy6FuSf8 z*j`gCAuA&a3W(MsO9J;L7Rj7A11;535u+)HQA~LH5U9(AK}-_1cZK%$_m}2~=^$qm zw$Z9fQP@aQ97x4f85%+CU;|zPVNC(+Wb6POkyP-0$w0O?(lzp7mM4UPrkMSS&``2Y za$jn4ket`@jFh6J0gMW)!54)==|hSR>v|O~Nw!>)RCGwq@Z~dkdo==MaWNQo66#Z6&FH)3(gNNUvB-rl~r86%nIhc&XNWqL4s4BKAO>IO=kO9;Uy zgh}_qxML|%>al1hRf-SDQdsSz^>;05iX>HCijN>L42&6?!gjc8Q4g|2)`JE=qA48J zlr!Dp;_Yk7U|3Q|h9nyJaFewt3<@n?ENLJ%xoLZ6%g&b0_RbwGPd(Mr(YbBYqE*>` zx?Ak>`Fh~!`}>=lEvb?%K@mwCr3BtHg8lt%{r&bJ^dN!I(bRLQ6hUq;%V!FV5b6P{AF}+@kr(iyV)5F{I_r6~Z;i5p6=L3CC#U+=?!g=S-!uUQno}K5NeV#JT z{*!%n&AQ9wvg^F7&nx@ofb2%CEW@4U`+O(3V1-8bR=Ueu(6Iz3fDo*Ow=zvW4RC;ZL0>0=YOK2%%c{UvlO8sNIb?e zC^Drebi#~)TM_7?!$n%frC5uM2n98vn+N~PIgXzng0$b^uW|x4wTxl?Jln%47jzyo`IhlUOJCzDU|&ekAe;&aM$i+WiXf3? z3LkJmzbOEEm@a)pjTyM%3@~B}Xh)s=ujzXQ5I{MoX-UoW+Qwlz={#cw!c4cqmb=A> z6l}@#h7o^aMmTJGtad%9ZN1|SL+1w!X8x z&ax0T)rjLY?gTf=ofTf=M){y{>1NS_jaU!-I+;MF}q& zV`fE$o6uGuv^W@~p&`*(hGzjED|9B=p9^j^!gmg}KAYg(sz zW@|%t{GmAkjepR|u-|HI=KgYP&nn?(^*qW*(aE8W7{>(nPpDialg54NCG)2bU@yVt z1c?#Lh|(6|^7L#0oZ$XX6p__gdf6f_Q4TteS(0>F#w-AMYMU|6GVma3L0(&nn@;=? zElCB!)&a_r<9NfP1h%l3Of>ePILGD7D=+WP?#^}GDSraudt1+Kf3N#m_m93i<@{;j zgTQR+$cZ) zZ;;Xj?gD?odBJ_bbD{RaiVG_*)Inkns5O#g*<~enaYb4vwik>E6pPoz+LB3v!eb-} z9l+wWXN@BF0iqnskC6p(aWNjP%a|hthzYNQOM8d?#f20rSw6~VS7d9m&a^A*mYwgq zvK}j>q`iZn7%p;W%=Q1Yma=r&aCLD3#Y&dXx@~=8{4EB-VwYlIzgTBb$Y7UD`>wa2 zN_SgFvvuQ;giZP4$_$_JWNI@jGAlFg43}|c>Y`-;rQQjjL2P7Kw(diZZBt#`M7cqV z%dT|!@H)F*x~F~1OU9nnrZwT0UG!(a=eh6t0cpIK0}+9N8aUlIpm`JCiNHi4?FAU| z0gMQE@-CD`+VwptjghC`DE6lXeZ0B`>+sRo6M$3$1Ec(ttr6*!s&6s+?4StBykI;5W*=Z7p9!V$EKW+ zbSb0>o2*n2<;hrv$w!pv!dwX}lSbI9lxwt(9hA-$C3e`P#AG3jW)np=hEh>TL>0+B zl#VK%rk185iSnYHB~Aqgk^9k$w3g<5O1oK{@DZC9rJT@h;eo+o0LxM#rEG0l6g5&$ zLWqi72rFhKWm6X9M9h`Tra(+Tj_3nPT?x|%0Mk#U9tIr1LCo}N%BVu0Boyt~qPR?6 z4*rIp|94d0DO-3nfxYr+@~-~jWjs3bu6(vacCeVeP8QPqu=8OZev{WKv%clqWPvOL zZAD*O8lQD%%WVsqaKMZ6%N<104tVjhtT!8g2Nx_4&isfl;hb=#v8HelvZeVne0@~F z;oE&o3t=jY+2zDvOB#Am(Aub>or(@B2x0}@8bu?GoUQ35;2+lk_zQVYmS!?ngxPr* zNuOk#y+PVjH(d;x2P@fhCLhOPK@Y2@r;yw=T}(=*D}gh-Zh9zTg>l*wG?6obRGH~X zSSc>E9Ons34HQk=M#~}J%ob^SES#V-7-p%`Is38zhc}8-2x--L%OkJ`I)t7ji9 zY|KB|oL?7wP==>DA6L8jK{;wO{*9U5$(M5n=PJ1R^_iYay&yKJgNk1EzHbN57t{^+Dk;1s8%PvW0E9!Yuz-| zHKHYv{W?H(hjP~RSO%<{d|2B}z4lNGf7%pID`Tbr8`r)?uY0K2N5y^=I%z7BCt^Qb zneAF|{}wK$585Gg7+>>fRUmk-&m%ni7bHK|z}0TLQ}fh~llQhfHM8ZZj3-~&aHn$L zl=EKGzL}BCA@$C5b_vWNKx zZvdG`l6U33fqUMyGv2ki-Wz>)z3rdX)ZMFTn5k))t$8xz`O3}JugPu9oqX^0Yp-8F zH8nW1u4}e#N2dI1XP|8JR~!}Vxe9Tr_tU`k+m8EnYjU0M?YOq%M$^=RpFaP=^RsJr z&erY9^nNXH69kuMOs;8`U>@{)C^Zzf<3VqpyF?&G}bNmv8>n=H@Gv zll){?Zp*J$ub=G6Hw1t3-22Z>Nw;cd8@5}v@Xrm;T=V9H+=+bOsht1bs&Z<`*2;7$4DAsJ46S#)0^IlGD z%pTA5WKxq#-d}b3c=mYi;Ek?Z+i&w9dq47iQa7{f(Dcsdepx-;aX54QQ~wc+6&t^< z$4;gL+y8Uk&8_OxYasleJD=>`DExe9ptsfii;b?{X3s+@ErH^O8#?l+^BmGd=wn@_ zRMuOFD$AYzfCq&_iVGNbLP!hCrMTe59UnUE$N-#J??gImLRgS`a^A$6=#m9?mT;w= zvXgKb&Vj{K#d0SG^bIC7hz@l`BP(AOyX5CvC6S=uqh*a$b~DfD9D%0dOa>6FgK^q|M` zSSzqV(^Hs-ojy&+h-|tkIV%~aGpmT7J3iJI9`sgVX=eW<12ufc}vqaF&*_vI`{$2S!`#x6xUY)Hv{E>RQdb0ZJ>MN^r zJEwXlSASZ){kEEMO;;Vh5ATusemA1X-hAzFNN%DD2C zE6Bg(&Sb`CD>qJiH~#+1GOqdvuTy;gi#0uaocLnk*xqUJ$7y+Y%1W?fxkf6D4sJ~` zQd+`#bR9eP1##}!v7lFDY#|)Ny!0Oe_E(43k^eho;ei2aWuSmjgA5FO%L+OdjO|C4 z@v;#)bOJvKKo+~4GJq7^kpc9DDFsMBHwc^laQIA0QVT7fa9B=8!eN%NGyV98k6%zA zsXvsIt?0vo5xHz85I_$rOBhj9H5^8I0Y8ud+(%LH?sOVv-xm&2!zhYx&T|hOK9~1D zR(W0Rb9J1%=1Ygu)iy7nHpgmy!L<#3wmFn<+m?S~eSYQYd~N+)e-H^_+{< z;L+AJ&UvgloK5RonN|0}r_PtN`q!^Gysk}i911H$*J^`U0x=+(0{g)Ljr@YvKyPbE zNv%)*A*)%pqOIE)qwgLX30R|y0YR^XEa_SwJya984iREv1;Hi~pSk*>_o2cWL&G7H zGavgikHG=iU|0|NT70JicNrTQf?#_ops8mpN$$}!kR-EqOWI8Inux<(XMZWS7M{ zbz3rVA~JQ$b?qWl>(&&c)GE*-3J{=fQJ^kbpj@vSZYB(1G${)IM^{c8_@DOoz1i7a z(d#B7apvuN-}~Ns-}k-mdhefWYCIgCnRkA7^!x1`_jCGTK6azPKYkB|c~0W`If<7n zQ9fqrxA0W9My)Yhzb$6(x5pg)4pwK2I{RH{vqjzg9`^P2`|!0#YhtziwG74)^~dV^ z>sZ+tt&cVIH?XoR+8Arg?K^}5=ku*i%iPGP(lfMdT#-1Bqql-MjM*V{@p#jS5FC1aZDCEuDsZ@UlR83 z?-`OsErl5+#kd>|26;O?iN6?%>e zC_KAg7*~Ug5STaPQ&N_M7%>ujxH^L6hod5hs>mr7tEl9~PF0XZO%#%foD>y<9D^zC zpvNN!J%T0Dv3rFmIC@E?q2O$dK5bfk(I1vXM2ZM%u5r0F&gOXP2 zHFi@}hSl;IG!AX)gS4wkP^=c5i-YSSP+LAoqP*{k_IN55ipt`+jK2H!JlPbIRcu31 zODJgEzkBx+t-&CZd(esYwXuD$2_XCu?b^t9#45jx7smUJx*OT;4%pO)iLHQ|GdyDD3V7>Ax)qmG$^fgVdhhcVSl`x_Dp z*^uO0!ib1X+r!jYVF?gv_nS>VQp= zJrZU%4!Vdb=q+fGFsa~flI(awNX5~nkZMZG;m8oFfFdOgeFN6SWm%G?pyyLENdaE> zigCy~YgPkR-F6|OYP#zMs9rKTy8UV@8I}70=qjqDGbqzVQAMrX{&~jjAztkmdfAz{JmoPl)VBI^FYc_RQk;^r7dYbZQFo8^fw7yl4X)G z*wePd^ewhdIj|N6H|4AtyZS18=*XUm8M`EF+9eq{9gO+sCQBQMy_JaS`&Nun|7Hrmh7~KDR%{YT^Y_SrQNhXTJe3CNz-4~!yIAK3;do6 z=&Cd@p{KlQPufi*rM*RpdarN+xA+Z+&@t=?aTq(MLyO2g%oz#87zPZL1zR>Sz&4?Z z{YkbK+Jfy*w&%KqaVV#R0yB#};~`xN;~S9x8x!HAF-V0Rj0B9CYHVI1A{_!)VrYnP zjwGUz;T45bu@w11P<80SkP(cGe%QXu0Yk7E{F-*)5gyB~;MB0Q8h3s=+JJ9I_gutq8-ejU=QaJE8K4 zI6Hwz5R=Jh1ayMZVRI`I06q#78dFe7a&0;W2I9$JRKBGq6!K4sE*~?S@kG2Q8i_-* zLYd1+=u)ECU0%0K+C)+wHbi6f`Kt=;u}^7(_kH@mcmU$ODMyA!G&SJXtx-9yTjELG zSsX^U%dw<3p*vw}MeUiVbq74C_^_?f_-gR7`gbM&jW_Dy{{5y2D_qZUwh= z8&OQ>mE}7RA7dCSJ80qVupJcjl&FgFgc+&g&0{wMZz1iP&$G60RB{j8(fC_ zypD7GrcZs#n>n`PtG#`0=3LgXV4a=H``RrQ{PEw}Ds=FIf9*Iy_A^R=1Q%<+QT zKWm@alsR5T*!0N1BXg$EunwS?UO!)GS~vUJtTKBlJGjueur)iJYieKIx_D{v^5TvM z#~-K%Rkq4=} ziAVKE|DoX@H~!tmk6y|jyOyiJmN~cL@!js2>6qQ~t**?8LjA_<>0JHx%!z6H>*osI z+RVAfAPg5SnE?E7_D!YuRV%#b!x7w=AG19;3oJ@0O)9TG}$;_m!W>^YC zS8jnRs%UY1#V8FXCUZ7V7_26#h#%sDQ3CG}WQ)X~!g7iHbgz z#^bB{*w^&&{`y&cX&hhLnf4H#v@7kVFEfobsP!7Pl5G-h2CJ<_t&i2ZD{B1}qpmAc z%q*pA(|+nxqI4aZ@v-X|u?`Y`61X9KQ}q>CTFk!XZ@NBR^J^do8}^IjNY|N?L1tL8 zn7^hP()FVo%Q(%_y<$OW(;CgXG+@afk8Uz+%3l-ST+tPb0dwt4^+HrHRt-5xhdz-= zD8`^_DF{`|=D8Rfmklq%n?Z!W=)+7YzRYe{gu+5crSshlgXbYF&#XmIIElE#h{F%> zD+ZzlB1J$3A%S!ZWpF4a-o*l9a?}NNERsZapg66sm)wYhC9l2|P#-xWya=1AQd~5| zA`#Q^XY0m1<0_)5nA*)|1#`}T%s#P%krjbuh7}wb=!VT_1;Un85L+U?-=!jV1Fs+; zh{H$#4I?NqmIx2PAO{jQ#v+-4H%%$?Fj8?<`0YKrZy~&bT}KjO1Ze>BHH!0OnBZtg zPdFLmR74YrB!M&XgemlZpCtTy<}Fmhb0|fOk@$FG42?!4#_+K$ff>s$1isVcn{RlJtrKVJK<5lT?lZAer=rOijHU-Kvd8mvA95{1Cy5Mo1?$5GCr{zd$$v&`#QZQ)!s znW=ZJ+1__N3*sO93;u=`Z_VwaGe@(>vish%yzjo_UTpuM`)>DNtNG4DdExL<>yfg1&=gz(xUTA-3G~X0h-2cJhyN7>d&37HndyZsIthhI2<%KJY+w<;S za69m8Xd%4#bl$xi-iXJ4yL+bltyJE#36Q|oHPba4UfBD{v!lvyOzY6hp;>j|_+rDI z(~rE}n8R;aSMb!#+GaXu)$EnIiJV7(&*AgWZikQ6xWK>NlJjlJ^nSU9{IJtAr)PV! zy>l1yzO8_%shd4E^K!O*VawZroNqfkGtA(?%mKjcgKySyr)9DI!JfOF55$Mf`7O`q zHa`DhW8T}FIa%oF`M`bGec$`${@JEkd2Zu!{gzz)mVEuTym$MO?eN1!yp%3*CS0`L zC~_QZfZ_EFp#ZOkA1!(v)M!4YZkP#P?ezr{tREhH%L6Moqts4gn+^%ilI)+*(;U~ zHw{LSG2;=DOp}q^W=b~rC0PPi{_$R*{zJYp@iJpgbKeF9Bt8PUc#r=M&r^be3zcMX zf(uys?m{?pyPCqlx+@fl#3NcLq`QgLAxg>xY*1cubV#{|5p;)SpLn%>YEGO#wyu{ zEm~24;QmwzChpry4wXG^ua<*Syc)5bJ(A;IahrdGPunFc86kX&St4YNoTF93iqR^; zgydx5(BZG>Q6+;Txo9-W#rg?l=Weu`LtS+q=G>>8@ya#1&3dfKlCetMlq(+4)*CCv z`Ye*i3>B0HgdjAHHBZ`Z;y~X@++MRDxVcf1nl+E|TUyB}1^vksuX!}VtQg&DmdfAK z%wCdw)%%ZbFGHK9aZY(2&^yiF<*($sUt`XBlx+2y$5h#)U27WfQ!52xqupO_y$fn7%`$&A-SxR4_t#>M5QxKs+@flbQ zLsm1S$TIRyH{pkAVxU!bsq!%M-F2sNEGR3awshOc^Pvk0?FZfSbvhyFRTOx5B(}OO zE=Az@sUsp(FLPg=aP4JzQr4X~au795Q9=MT6t5?u(s8jE56ilpolEGJByw99Y0+ot zoI7RHOo<&t%(Lb5Plu0RojG z-wapq*H7CD{&m@%%dMR`{P%Y*xjL8pohyy)(+uX60=C*AAV(X*krwc7xms`4W zEnSPp^DVoI9Zuw1PcAi`Ty=0wE%Qged-T202Ms@H`Ky*6?fu&$KRohi$JviYm#)9M zd|k?2mzJ-O=J3C7w9vNsd*|Lhx7eROmv1|;+;$|_b|l~S{BqlwT-%v{QU3MTKiyht zJCkpFd9L@9Het5+7ftKu&(ED-;NN%MaXsK4ZuxNk(#tmrwVZ-T^WQ0@G_C`B z%88^<$tA63H`CUP4;q~|$eJh4!+kA|QGuhBo-2c=ZTCy5xeCZC7PM?%hOJX>j9bN` zD(yIL;YO=jPId+(*}tOIaTrr7>+UH}{9tv+QQF{Z9tjS*X@lUJRUN--KSyOhiO2a$ zncmeBPgOA^`Q#*XJY<38CVo^&Hh~`&W;JjsBe~v1a*o_coYHvH6|zu1Phz2lJM|n` zc$KaWz;!)N@jbLGQ}|+3!F`vBp5p$Okl%Rnj)ZZBY^H!nsk}%xKum4U@(nm19T>PA zj3$PQHv{N!h7u7tdqGOVyeom@ROop&ES>Ea&q4^LHbS$&0SG1gahn6!fI*QFd*e)j zVj$zv073Pr;*e~dZ_@!Snk5{_O0w!9O!OdM5Z}d;26T3WS>edHc#;Fm9W1{gE8R$U z;BY%+9OvMe*RU5Mrn2eur-YA=XdSQ!kp$8KbQwZ-7#AR}>h)i`55e*y0lV&}gMT(u zh~-gq56dTz0akUdQ8uiDodprPn^r@|>uR7zw?*l!*g6?WDk(^WUdv*Vka^uhZ+`N( zF@&nu5({WqDR%0t*Bd0Ps4B(`bWJ9vvM5itCKE}+ariw974R8mX98t~DwOZg1bj#h z4dOJOq0)Voq*GoZ0E?t5Ndo^Sy=JJ&S)}xrPInBG)D&Vu6e59OR*6V6xoxdwssi*q zU{~9ifm+?n`C68|PZc(9n!d2o)beg)R?4>Ko3_umr!CXHpF6mkx@F(yoNsd>cw#Ad ze%7{J*ZFZ>=hCI2$5u;CZ3VzW^pS7hiqF67+k#BRLU-P`Yt`#$_Ds81YdK%t?W5m3 zy6WQEyQWXxK0k9l>jF?=*FNLd60D1_d!HP$tXuti!%nR8w3(_NBSD{?9)VAwE z(<9$Ag{GG2-d{8{&j;oL#pZ&qcKWrM-c`H1(F2>b|0KWk^!e<`<+j})x9wiKGyq#! zXxUU~SYKH8)MtJIs^k34pVgx@?Obi*>egp{?@5dM|7>i@zh}v{$1owKa0w(M$AE=x zt0>Uf&PnbDVtfQ-Rr4KbTZA)};s2NK0G10J#|(`fL6h|z7^im-2Rz2^dJ7P2yFjrh@=ydm z+?X;+-K^NXa75uZ;$$|N2#@G|OeKV**^I~7KAw=d+11GnYd1})DTkgaAt4L{z>;s7h%-w1Px%ADDa19@W~e-m zmu_dr(8?m!7dcDrdbNJaEovZDsW8<_y0R)?twRxQvSAHFQTG$-Fp5`(!)`Aty@J1S z*}o;{-?FfC@nGKn4E*)Z6N}rIJ5S^~k$W_CwtvQ-^)1N%=IUmg3-DJFSYD%vQ^X(O`#=J=N+Kg=@ac}k z7n|sI(<|&s(<>Do1njD&+_(|2_4R#9sf|9+iLL+pgpla_MU8nQ>J>yoH<-CmKA=`c z^F36$kJse3GV-De(=AAmu+}ZGp^b5P9s82)tVwsNTKQA-#|Z$!H^QX+4qgF!U!U@q zDD?FOT;=r`tYgEI-CHw}fQJ%tv^kNG^Mpdh%Y`A`7YgAFC|YcBhC)&z911B{2}Oop zBrZ%&lzS-YHN|9ISdKiw%XxS!b0{Q)s8RhN9)d2CzV{B(6j2Ml80O0y6UA;9p~wJ zT#wS{67R5IS`GGAa?W|M8+^$cT#t&|i3PeTD8z zE8+w@hf#h&%V5Vcti-5KC6Js_(nkAprP9Gti9u2xB+#n-4L(Hep%h&{23fJIA$Z6V zAf2F)g;U7HC=RN&v&hOhX-SZM^#U%K1j&|_M)^nljjXzst@8s zj&9=_`GfVWy?70!cz=|xjw%$O={2m?yt!%^0!RZqJ50NiCV7@#K6)`0lObUwG?cpQ zupv@MmA^*=$%A?YuT?A0^PgGTdD}1hc#hxppIk?d>-ag>@Kdhwr`)>#;CB3s+xIiB w`)Az7PrMtmuP%AIGtS43X5RmJ_h$ace;w!T{8OLTrFgz)_T^u4^kswoALT?X-~a#s literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py new file mode 100644 index 0000000..d3f051f --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py @@ -0,0 +1,189 @@ +"""Seed-ensembled LightGBM that early-stops on cross-sectional RankIC. + +``RankICEnsembleLGBModel`` wraps ``RankICLGBModel`` (per-day RankIC feval + +``metric='None'`` + ``first_metric_only`` early stopping) over a seed ensemble: +one sub-model is trained per seed with identical hyper-parameters, and +predictions are averaged across seeds. This is the model class the +``tac-rd-rank-ensemble-isolated`` reference run wires into its workflow +(``module_path: tac_qlib.contrib.model.rank_ensemble``). + +The ensemble inherits the RankIC early-stopping behaviour of the single-seed +model (valid RankIC drives the stopping iteration) while the seed averaging +stabilizes the prediction against any single seed's early-stopping path. + +Training is parallelized: the seed sub-models train in a thread pool — +``lgb.train`` is C++ and releases the GIL, so concurrent seeds do not block on +the GIL (5 seeds ~40min/5 on this box). Measured on a 6-physical-core / 12 SMT +host: the seeds scale ~2x, not linearly — the runs are memory-bandwidth bound +and each Booster caps its threads at ``cores // workers`` so 5 concurrent +boosters don't oversubscribe; larger-core hosts scale better. The qlib data +pipeline is warmed once on the calling thread (fills the handler cache), and +each worker then prepares its **own** ``lgb.Dataset`` (independent handle, so +no concurrent ``construct()`` on a shared handle — LightGBM's ``Dataset`` is +not thread-safe to build). qlib's ``R`` recorder is also not thread-safe, so +the per-seed evaluation curves are logged on the calling thread after the pool +finishes. + +Wired into a workflow yaml like: + + model: + class: RankICEnsembleLGBModel + module_path: tac_qlib.contrib.model.rank_ensemble + kwargs: + loss: mse + learning_rate: 0.02 + num_leaves: 31 + n_estimators: 3000 + num_boost_round: 3000 + early_stopping_rounds: 200 + min_data_in_leaf: 20 + lambda_l2: 0.5 + colsample_bytree: 0.8 + subsample: 0.8 + subsample_freq: 1 + reg_alpha: 0.1 + reg_lambda: 1.0 + seeds: "42,7,2026,99,123" + parallel: 5 + +Any ``**kwargs`` other than ``seeds``/``parallel`` are forwarded unchanged to +every ``RankICLGBModel`` sub-model (same params, different ``seed``). +""" + +from __future__ import annotations + +import os +from concurrent.futures import ThreadPoolExecutor +from typing import List, Optional + +import pandas as pd + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP + +from tac_qlib.contrib.model.rank_gbdt import RankICLGBModel + +__all__ = ["RankICEnsembleLGBModel"] + + +class RankICEnsembleLGBModel(RankICLGBModel): + """Seed ensemble of RankIC-early-stopping LightGBM models. + + Parameters + ---------- + seeds : comma-separated integers, one sub-model per seed. + parallel : number of seeds to train concurrently. ``0`` (default) = auto + (all seeds, bounded by the available cores); ``1`` = sequential. + **kwargs : forwarded to every ``RankICLGBModel`` sub-model (model + hyper-parameters). ``seeds``/``parallel`` are consumed here and not + forwarded. + """ + + def __init__(self, seeds: str = "42", parallel: int = 0, **kwargs): + self.seeds = [int(s.strip()) for s in str(seeds).split(",") if s.strip()] + if not self.seeds: + raise ValueError("seeds must contain at least one integer") + self.parallel = int(parallel) + # drop seed/parallel handling from the base kwargs, keep everything else + self._model_kwargs = dict(kwargs) + super().__init__(**self._model_kwargs) + self._models: List[RankICLGBModel] = [] + + # --------------------------------------------------------------- helpers + @staticmethod + def _cores() -> int: + try: + return max(1, len(os.sched_getaffinity(0))) + except AttributeError: + return max(1, os.cpu_count() or 1) + + def _worker_count(self) -> int: + if self.parallel > 0: + return min(len(self.seeds), self.parallel) + return min(len(self.seeds), self._cores()) + + # ------------------------------------------------------------------ fit + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + """Train one RankICLGBModel per seed and keep them for prediction. + + The qlib data pipeline is warmed once on this thread (handler cache), + then each seed sub-model trains in a parallel worker thread on its own + ``lgb.Dataset`` (LightGBM releases the GIL in ``lgb.train``). Evals + are logged on this thread after the pool (qlib's ``R`` is not + thread-safe). + """ + n_round = num_boost_round or self.num_boost_round + n_es = early_stopping_rounds or self.early_stopping_rounds + + if len(self.seeds) == 1: + m = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + m.fit( + dataset, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + evals_result=evals_result, + reweighter=reweighter, + **kwargs, + ) + self._models = [m] + return + + # Warm the qlib handler cache once on this thread so the workers' + # concurrent prepare() calls only hit cached frames (no first-write race). + proto = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs) + proto._prepare_data(dataset, reweighter) + + workers = self._worker_count() + # Cap per-Booster threads so concurrent seeds don't oversubscribe + # (LightGBM's num_threads=0 uses ALL cores per Booster). + per_booster = max(1, self._cores() // workers) + + def fit_seed(seed): + m = RankICLGBModel(seed=seed, **self._model_kwargs) + if workers > 1 and "num_threads" not in m.params: + m.params["num_threads"] = per_booster + ds_l = m._prepare_data(dataset, reweighter) + booster, evals, names = m._train_from_datasets( + ds_l, + num_boost_round=n_round, + early_stopping_rounds=n_es, + verbose_eval=verbose_eval, + **kwargs, + ) + m.model = booster + return m, evals, names + + with ThreadPoolExecutor(max_workers=workers) as ex: + results = list(ex.map(fit_seed, self.seeds)) + + self._models = [m for m, _, _ in results] + + # Merge + log evals on the main thread (qlib's R is not thread-safe). + if evals_result is not None: + for m, evals, names in results: + for k in names: + for key, val in evals.get(k, {}).items(): + evals_result.setdefault(f"{k}.seed{m.params['seed']}", {})[key] = val + for m, evals, names in results: + self._log_evals(evals, names, prefix=f"seed{m.params['seed']}.") + + # -------------------------------------------------------------- predict + def predict(self, dataset: DatasetH, segment="test") -> pd.Series: + """Average the per-seed predictions over the given segment.""" + if not self._models: + raise ValueError("model is not fitted yet!") + preds = [m.predict(dataset, segment=segment) for m in self._models] + if len(preds) == 1: + return preds[0] + frame = pd.concat(preds, axis=1) + return frame.mean(axis=1) diff --git a/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py new file mode 100644 index 0000000..d03e661 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py @@ -0,0 +1,238 @@ +"""LGBModel variant that early-stops on cross-sectional RankIC instead of l2. + +Standard qlib ``LGBModel`` early-stops on the regression loss (mse). For +cross-sectional alpha signals the quantity we actually care about is the per-day +rank correlation (Rank IC), which mse early-stopping does not optimize for. +Experiments on the 50-ETF lake (SP-5d 55-feature panel) show that early-stopping +on a custom RankIC feval lifts RankIC 0.047 -> 0.075 vs. the mse-stopped model. + +This class reuses ``LGBModel``'s data preparation but: + + - tags each ``lgb.Dataset`` with per-day query ``group`` sizes so a ranking + metric can be computed per trading day; + - injects a custom ``feval`` (mean per-day Spearman of pred vs label) into + ``lgb.train``; early stopping then selects the iteration that maximizes + RankIC on the valid set; + - forces ``metric='None'`` + ``first_metric_only=True`` so early-stopping + tracks RankIC only (not the regression loss). + +Wired into a workflow yaml like: + + model: + class: RankICLGBModel + module_path: tac_qlib.contrib.model.rank_gbdt + kwargs: + loss: mse + learning_rate: 0.03 + num_leaves: 31 + n_estimators: 500 + ... + +The rank feval is used for early-stopping selection only; the objective stays +the configured loss (default mse). Set ``rank_eval=False`` to fall back to the +plain LGBModel behaviour (early-stop on the loss). + +Generic: works for any cross-sectional panel whose qlib dataset index has a +``datetime`` level (each level value = one query group). The per-day groups are +derived automatically, so no universe-specific configuration is needed. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +import numpy as np +import pandas as pd +import lightgbm as lgb + +from qlib.data.dataset import DatasetH +from qlib.data.dataset.handler import DataHandlerLP +from qlib.contrib.model.gbdt import LGBModel +from qlib.workflow import R + +__all__ = ["RankICLGBModel", "rankic_feval"] + + +def _group_averaged_rank(values: np.ndarray, gid: np.ndarray, offs: np.ndarray) -> np.ndarray: + """Averaged (tie-corrected) rank of ``values`` within each group, vectorized. + + ``gid`` maps each row to its group id; ``offs`` holds the cumulative row + offsets so that group ``i`` occupies rows ``[offs[i], offs[i+1])``. Returns + the same result as ``pandas.Series.rank(method='average')`` applied per + group, but in one pass (``np.lexsort`` is the only non-linear step). + """ + n = len(values) + order = np.lexsort((values, gid)) + ord_rank = np.empty(n, dtype=np.float64) + ord_rank[order] = np.arange(n, dtype=np.float64) - offs[gid[order]] + 1.0 + sg = gid[order] + sv = values[order] + newblock = np.empty(n, dtype=bool) + newblock[0] = True + newblock[1:] = (sg[1:] != sg[:-1]) | (sv[1:] != sv[:-1]) + blockid = np.cumsum(newblock) - 1 + block_mean = np.bincount(blockid, weights=ord_rank[order]) / np.bincount(blockid) + out = np.empty(n) + out[order] = block_mean[blockid] + return out + + +def _per_day_spearman(preds: np.ndarray, labels: np.ndarray, group: np.ndarray) -> float: + """Mean per-day Spearman rank correlation of preds vs labels. + + ``group`` holds the number of rows of each trading day (query group), in + order. Days with <3 valid rows or a constant pred/label are skipped. + + Vectorized: per-day Spearman == Pearson of the per-day rank transforms, + and the Pearson moments (``sum``, ``sum`` of products/squares) aggregate + over each day with ``np.bincount``. Runs ~10x faster than the per-day + ``pd.Series.rank()`` loop that preceded it — this feval is invoked on the + train and valid panels every boosting round, per seed. + """ + if group is None or len(group) == 0: + return 0.0 + offs = np.concatenate([[0], np.cumsum(group.astype(int))]) + gid = np.repeat(np.arange(len(group)), group.astype(int)) + rp = _group_averaged_rank(preds, gid, offs) + rl = _group_averaged_rank(labels, gid, offs) + n_g = group.astype(float) + s_p = np.bincount(gid, weights=rp) + s_l = np.bincount(gid, weights=rl) + s_pl = np.bincount(gid, weights=rp * rl) + s_pp = np.bincount(gid, weights=rp * rp) + s_ll = np.bincount(gid, weights=rl * rl) + cov = n_g * s_pl - s_p * s_l + var_p = n_g * s_pp - s_p ** 2 + var_l = n_g * s_ll - s_l ** 2 + denom = np.sqrt(var_p * var_l) + valid = (n_g >= 3) & (denom > 0) + corr = np.where(valid, cov / np.where(denom == 0, 1, denom), 0.0) + return float(corr[valid].mean()) if valid.any() else 0.0 + + +def rankic_feval(preds, dataset): + """LightGBM feval: mean RankIC (higher is better in lgb convention).""" + labels = dataset.get_label() + group = dataset.get_group() + ric = _per_day_spearman(preds, labels, group) + return "rankic", ric, True # (name, value, higher_is_better) + + +class RankICLGBModel(LGBModel): + """LGBModel that early-stops on per-day RankIC via a custom feval.""" + + def __init__(self, rank_eval: bool = True, **kwargs): + super().__init__(**kwargs) + self.rank_eval = rank_eval + + def _prepare_data(self, dataset: DatasetH, reweighter=None) -> List[Tuple[lgb.Dataset, str]]: + ds_l = [] + assert "train" in dataset.segments + for key in ["train", "valid"]: + if key in dataset.segments: + df = dataset.prepare(key, col_set=["feature", "label"], data_key=DataHandlerLP.DK_L) + if df.empty: + raise ValueError("Empty data from dataset, please check your dataset config.") + x, y = df["feature"], df["label"] + if y.values.ndim == 2 and y.values.shape[1] == 1: + y = np.squeeze(y.values) + else: + raise ValueError("LightGBM doesn't support multi-label training") + + if reweighter is None: + w = None + elif hasattr(reweighter, "reweight"): + w = reweighter.reweight(df) + else: + raise ValueError("Unsupported reweighter type.") + + # per-day query groups: each trading day is one group + if self.rank_eval and isinstance(df.index, pd.MultiIndex) and "datetime" in df.index.names: + group = df.groupby(level="datetime").size().to_numpy(dtype=np.int32) + else: + group = None + + d = lgb.Dataset(x.values, label=y, weight=w, group=group, free_raw_data=False) + ds_l.append((d, key)) + return ds_l + + def _train_from_datasets( + self, + ds_l: List[Tuple[lgb.Dataset, str]], + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + **kwargs, + ) -> Tuple[lgb.Booster, dict, List[str]]: + """Train a Booster from already-prepared ``lgb.Dataset`` objects. + + Pure training — no ``R.log_metrics`` — so it can be called from worker + threads (qlib's ``R`` recorder is not thread-safe; the caller decides + when/where to log). Returns ``(booster, evals_result, segment_names)``. + """ + if evals_result is None: + evals_result = {} + ds, names = list(zip(*ds_l)) + + callbacks = [ + lgb.early_stopping( + self.early_stopping_rounds if early_stopping_rounds is None else early_stopping_rounds + ), + lgb.log_evaluation(period=verbose_eval), + lgb.record_evaluation(evals_result), + ] + if self.rank_eval: + # early-stopping must be driven ONLY by the RankIC feval, not l2. + # metric='None' suppresses the default l2 metric; first_metric_only + # makes early_stopping track the single remaining (rankic) metric. + self.params["metric"] = "None" + self.params["first_metric_only"] = True + feval = rankic_feval + else: + self.params.pop("metric", None) + self.params.pop("first_metric_only", None) + feval = None + + booster = lgb.train( + self.params, + ds[0], + num_boost_round=self.num_boost_round if num_boost_round is None else num_boost_round, + valid_sets=ds, + valid_names=names, + feval=feval, + callbacks=callbacks, + **kwargs, + ) + return booster, evals_result, list(names) + + def _log_evals(self, evals_result, names: List[str], prefix: str = "") -> None: + """Log recorded evaluation curves to qlib's active recorder.""" + for k in names: + for key, val in evals_result.get(k, {}).items(): + name = f"{prefix}{key}.{k}" + for epoch, m in enumerate(val): + R.log_metrics(**{name.replace("@", "_"): m}, step=epoch) + + def fit( + self, + dataset: DatasetH, + num_boost_round: Optional[int] = None, + early_stopping_rounds: Optional[int] = None, + verbose_eval: int = 20, + evals_result=None, + reweighter=None, + **kwargs, + ): + if evals_result is None: + evals_result = {} + ds_l = self._prepare_data(dataset, reweighter) + self.model, evals_result, names = self._train_from_datasets( + ds_l, + num_boost_round=num_boost_round, + early_stopping_rounds=early_stopping_rounds, + verbose_eval=verbose_eval, + evals_result=evals_result, + **kwargs, + ) + self._log_evals(evals_result, names) diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py new file mode 100644 index 0000000..4afcf90 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/__init__.py @@ -0,0 +1,3 @@ +from .optimal_stop import OptimalStopControl # noqa: F401 + +__all__ = ["OptimalStopControl"] diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ad10c2ebe37c16417e67c7aeb731ad1fcb6da2f GIT binary patch literal 234 zcmX@j%ge<81f8>6vtohtV-N=hn4pZ$VnD`ph7^Vr#vF!R#wbQch7_iB#weyrW=)ot zj6g|E##=)E1tpoei8;X~`327Tc_l^pIYrDsDL+l-TRiy?x%gtBBv|wodwhIiPELIM zN`}uMTYeeqCl(awmn0_Z7UpCoff(^%Ml#4Kpg?g+QDRAIdZm7Rd}dx|NqoFsLFFwD ro80`A(wtPgB2J(gAcqtS0ErLGjEszT8FU_S%U|G9Xk;&92MPiJ@PIxK literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc b/code/tac-qlib/tac_qlib/contrib/strategy/__pycache__/optimal_stop.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d684b3216b040071d9ee4fa920a0e0c7486d278 GIT binary patch literal 10428 zcmcIqYit|Wm7d{4;+qs7qTU=?4~y2zmhCuk{E{szajHm3ByVD-S&HI}BuadFW)xfM z5O4}-OQTMsG+kIpvl|r~ph{zO6<~L(0tM<|i#9<2a7?FUXI-opMX?ALyHM&BE&6NE zxx*o8Md@Q7*YcfnUuVue=iKk!IhQ}R+btBNYu|i%;inxG^-C;h$*2l22MfzXDyF;oL}#&~VQ6f%*znz%V(30V@>kd>5cmao3)2LtRv*rQIAoq^D6Y>?C zE=7p)UW~+-$SmhmaTd)N6D-LT6Hg`QU?P`ba`R-Gd`wsj_AwLBFp*giTBfHHk;`Em z6lNpK0%{KYfFojp3(UrnF_E*(r})@PDya@6CPD$qmV2sujG5uNNOYbPfTIvcHZVP~ zqUtwgHCxJKP|GtMXq0FuA~JCfTn*)kC9CBDZ<>s)s51g_E`#c(r;{nw<={Qw5EVgs zxQk2R!zi`tzJ+3XD0zr_!k<=nxU+`$5O4sH((yZVB`Q{thpF zHpauNVo5Q@T;!LMsKUirFoeJmR*~c~$Z+r!DoGh8&qv(wWS zP{M3%E;ztE8RPjBFAzfvKXr~-5+ZZNXcyzLnZ2mGD(NbGk3z!)&vN|q^q{3gj>MqE z%<`!OG~)1qnMia|xzuEX%bf78P$vC2xw*Vr{>;#1JlPR)f(#F$&V3m zKh{Aux7GIYe>}bQ-5*9t?^0%0pY;7zAie^)%TXa3uvf>@Aw8>O^_+n-ay48nXX4DP z0c!A{@hzK~`a}O%Tb7!o*_x}4kcG3dcDDAa;l0u;LpIjInywl`cGk(7;pt#qtOcG< zwvM%4r9-YPb(s#;O;SNOZ-zmUBOw20U>H|$KY_+F_5#`g=lEcw|tgQEi+RG^uM zbx76^te%Od&WO0<6Kp?;ylj@_+|z-A3j31R5S`vJ(K2RQI6?3cAVJ{7|h0m$V?p3X_ncq zvK!}d0nE*k^j{}~Hl;@0FckyZS}~VRsF*YjD)tAo2gPdSRZQB%6rB)PthjN6S#FN! zIK}ZGekj&B_aYY$$C4nqVvD8{iBwW8Md{LNu))&=|E(0IF2cX2Z&0GvN?mhE)GKfb zVRaHMStP4uW9hdHlKma}Ycx$sI*AJ7;wDMbLrGsQ8K49s)g&3AR61f&lvD$ynsQq$ zlxoW*6O>RQl9s8UemrR45hwV1%>0-&K=v*bG%H473EW#Tg<)FYe_`H%%}&f*nAKtC z#>|767qXyJF$i3IR^|D}`|~~=`QW>in#D^Ieoj!#;czq_5rlA9z?b5ptfVWyKQb6e zrw7GIv>&$uB*KIs+b>jT&@os#r-ae-2GYxX3ou)0{w&lEXcK)HItVk6{q73&=^o1L zxK=Bh8n&q#v!&>1EH(#=4IRbCu20NRv2CH8zF%9Ba!|g;tl_%3*tiECj$#7?4|}n7 zU)FHP)Kq?!s@>kK;g#C^@D{4c^*G zN72J65ceTG>)|#;fJDGv49`Fg3WXkFY+h;EZTed41?ViGO|MY*oZeSguC1)G*_FJr zBWLR1+o3*M_Dd*~1s0(I0@Dld+?f{2zFTQwX>AG_ous=`2P4Z~yn8RP;>p$1*NrPi zf)+u8V$$|}MEFI;NaiLWz~F*YRbmRKlW}yQ-Iq?Ok%ch= zvbsVQUEYGLU3RsvcjR5&S>w;0-ZkU8?@dS1Q(ts?R=I1^T2ywn=S=OY3aX8d00wHH z@4p4a4LFa$Isj^C+Rd#1Okw(x;U@ha&H&OR)j^3RnxZ->z8&E8&*TT|tz@F8?gDVU zL|xWjpf1s{_TB}rQ(Y{^Di%SE@M0LZW7u_)tV)?#I2n-r?Oi?p5QxNsr6?YO20Rh* zPpOLuB}c`L2KPF}JhhzW&f@VoXjI+R5KXZhtZaOTq6hZ;BQRD#&7q~r-V3}mF1veg zRR`27LAxu|w$5mtr;8qc!P6~!x(l9%WzWOK#@3Clt&R^HxAm0wk^7X+>)57rcE|5N zt)X0vzozKV4!PBtn|v-8oy~b3&U@x=+vc|Q*z5NM$G_axPJL_duvPzog@*hAEWy2r zm|X<#C7urzJJW7P*AAe7B)c27+PO*$r80(m5%4Ud&%my_tCQ_ImOyX`2g!J226oS}1l;3oB5uJX&B3D3xKH8&!Q;S=R2iRvv&632!q@Id zW$T#5X03@eYLYA<{_IG=yKP4m6txi+R~P>=+EWzSls_Q5n0qR~xUrC*uHFoRW1RsI^p{TAvYWm3|>a zQN&Md85c;>Tj`}yl1(z-tko=#sRJGKX>FC~!eE8gC>RNVC+^qE3qu-JdEPW>aF}sR z?w5=b_@v|a6G%ls+mXP}z{{aWfb@8D zM>7;9m*m#o1Mk>{$6^y$(n};jW9&#Mflw!TVC;zhgp5x-pm78p z`~CziTiz?7MW+69>^H1&tURT9&G+~L$-Z!=(xOq79z=_~`NdhSZ{;b~OO~5fO}8-8 zvG7=>Rii2^fCfLk{ukgGPRWLfhSCcL3UKC}*0=IxZ8zBM#;bM zL}ipl!RlNBn>JM;Q=>Kx^fspTt31)lJOqoA&r3~G!;5gvxxVLf{I;p$x4>(>!b79b zCt&<(Xnk6%t2}`}uBK9^no``4DOe=Ut+aHzwUnnLl3K}G z*-`R}qY?viMhV-cc0e!K3wT!*w(jB$NoQ62F4&vdQ;j24GPIXv0FRUQ-27jn2YBNf z(w-_=c9E6r`Z}pCN&kN$&Glb^r!>MU|1s#X2|Q*u#M*5q`xDk))lSwSb-ZMeAjh3d zFiIU@FW4(!Z661lcMvOfe$LzRt8GV8SN=_>cGS;wea_hMj)72=I;AenzcZax_HKff z2Y2UPtnztHlH&0;PVJ5=!cz)pV00N&XcuG)e)bH)8(fFwXADwtk8iirz zhyb@*cyBZJCGU41`?r;EoaB#Um;G9-O$*>F4*ZH&<8Ok>-Mw2lYFT zjAgk9d`n|+fd|+Golc$|8{_w(KqR2%Md~1CB*=!xe8mbkO~4ASS0E@?1F>=pD2p{^!W?wyzxSa#NiY@#70#;&O$IY#Vbub z$jz1_C*hgpk`0z192o3YzepYGy$BbESng6>R4i3yvT8Jbt(w&W5jqXmOz^Rb)h0q$ z)nfFU$1}TP$B*bRgdm|S7~NJ1)Q3U2X~k3uSquDe>}!RnV_emZ06obyV^~Zkf_7OcCRP-;PL zu&kh1MY#QmtGZU4YHhe2K(@>*Eo%blL3G8=#%5ScZui5gY^ z0y-!8uK%N%8l&;gaY+s@t4oH`ZpBvi?GnNIoW26QQ}_aSB}8z1jRjwi?CUA`j>x_v zdEX;hOR>GH(0)j6KU8QxEw`V(bu`~TE}Po2*3}EeKt~}kBnO5Hfm3qe)UBr6KUe-tfwe1KB5vR$tEA`i^Hk^ml>H;oI$p?|SyE53P%NPhZZ~chBK|HF+($ z7Roz%vif2}OQE4xZs;vE9FiLjw^AnRlKjw)d}^*1TVLY#S-x z5CncZyR+82{-)QLvbLhLE^8<@w!hu>Mq8mVcu_Qy8a@6EkGcZ+_15em21pCmp=kt4}b97-TKD0zCzOxx#`H3A>VX7 zUw>j%f7jc%erQvF%UbN{`+@O?mhW3W)ziKadX>Ici53mXo}tYHcVJT9K*8HBd%M>! z-th*D-iCsAuk77h@D9md=&>c`yhC@qV?|d>DNySDnXCI=Yx}BUb?SA?b{EyOZ@Z83 z0h4{Qci-muoOfT|`)JmD*VnM-efq00?%RZq)}lixU*J%a{2{MD7Uskbk@apAVJ z`{yoS5vmqsCxcpZx?jC??b52Sb|&v+)(?IA#J5h|b{-)5gg{rP?CD&e-00n0{!!0Q z`abBpo7&#q_r|`z?Ys5_xI$;q=6~zpnp6lJlLN=Lrt*Q)`Q|gX zSh@KL+4jV>FM!pX_X3ON?hQks=V2NCnjZea!L5rw>bN!c&wcqvC*-DytP^^4biwQ& z}}^! zoMdy)#v!?RFgsT4Isogc>+oB~x6N;we{lGRN56kGd+x5wwZ6dLHcdG_zmy?gF<;P7irTaOl8an7ew>c4bDfuc!R%-ia2=F= z8Y|#Xj~wb9*-_w7-=u-t8QFK{*64?$dEaDq6elfgJhZh4^ZAFf-#hpIADpwRsi4NV z+dp!3ZnwfjK4mCZVC_pAAvthl%L}lPGo2vu?VyWn#EKq%U!mTj;h9*yN+T56>i-Js z$Qgi87`Wp}D>)Jn%BR&q=)Mc+-<#9yz$C}VV56$xIdO?k@?U{^IJ#x4x{_^=FsRh3 zJru)CDisgX<3Tsx_Ti6&;P?Yap+t&>pY#B23y1MXCZ(3zaF|U+p}KD5?C9|Mv8nLc z@u_Dngr7PyrMSvf&z+khH9oE8$>Hb1k57z^ghz%iOe&S?bK_Oj3Zx?e*;K@3VZ+n literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py new file mode 100644 index 0000000..79aaad9 --- /dev/null +++ b/code/tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py @@ -0,0 +1,217 @@ +"""Optimal-stopping / stochastic-control strategy for cross-sectional signals. + +Entry is a control policy: a symbol opens a position only when its cross-sectional +signal percentile is at or above ``entry_pct`` (i.e. it is one of the top-ranked +names) and the portfolio has fewer than ``topk`` open positions. + +Exit is an optimal-stopping rule: a held position is stopped (closed) when its +signal percentile falls below ``exit_pct`` (the continuation value of holding is +no longer worth the risk), OR after ``max_hold_days`` (time stop / finite +horizon), OR when the position P&L breaches ``sl`` (loss control) and the +position has been held at least ``min_hold_days``. + +Sizing is fixed ``notional`` per position (equal-weight control), unlike the +TopkDropout cash-allocation heuristic. + +Wired into qrun workflows like any ``BaseStrategy`` (see ``PortAnaRecord`` +config). Mirrors the API usage of qlib's ``TopkDropoutStrategy``: ``Order``/ +``OrderDir`` from ``qlib.backtest.decision``, ``trade_calendar`` / +``trade_exchange`` / ``trade_position`` injected by the backtest executor. +""" + +from __future__ import annotations + +from typing import List + +import pandas as pd + +from qlib.backtest import Order +from qlib.backtest.decision import OrderDir, TradeDecisionWO +from qlib.contrib.strategy.signal_strategy import BaseSignalStrategy + +__all__ = ["OptimalStopControl"] + +DEFAULT_NOTIONAL = 20_000.0 +DEFAULT_ENTRY_PCT = 0.80 +DEFAULT_EXIT_PCT = 0.50 +DEFAULT_MAX_HOLD_DAYS = 10 +DEFAULT_MIN_HOLD_DAYS = 2 +DEFAULT_SL = -0.06 + + +class OptimalStopControl(BaseSignalStrategy): + """Optimal-stopping long-only strategy over a cross-sectional signal. + + Parameters + ---------- + topk : max number of concurrent positions. + entry_pct : min cross-sectional score percentile required to OPEN (0..1). + exit_pct : held positions are stopped when score percentile < exit_pct. + max_hold_days : hard time stop (finite-horizon close). + min_hold_days : minimum holding days before stop-loss is evaluated. + notional : $ per position (equal-weight control). + sl : stop-loss threshold as fraction of entry price (<= 0), disabled if 0. + """ + + def __init__( + self, + *, + signal=None, + topk: int = 10, + entry_pct: float = DEFAULT_ENTRY_PCT, + exit_pct: float = DEFAULT_EXIT_PCT, + max_hold_days: int = DEFAULT_MAX_HOLD_DAYS, + min_hold_days: int = DEFAULT_MIN_HOLD_DAYS, + notional: float = DEFAULT_NOTIONAL, + sl: float = DEFAULT_SL, + risk_degree: float = 0.95, + trade_exchange=None, + level_infra=None, + common_infra=None, + **kwargs, + ): + super().__init__( + signal=signal, + trade_exchange=trade_exchange, + level_infra=level_infra, + common_infra=common_infra, + **kwargs, + ) + self.topk = topk + self.entry_pct = entry_pct + self.exit_pct = exit_pct + self.max_hold_days = max_hold_days + self.min_hold_days = min_hold_days + self.notional = notional + self.sl = sl + + # ------------------------------------------------------------------ utils + @staticmethod + def _pct_rank(score: pd.Series) -> pd.Series: + return score.rank(pct=True) + + def _entry_price(self, pos) -> float: + # Position stores avg entry price under key "price" (see Position.position) + price = pos.position.get("price") + if price is None: + price = pos.get_stock_amount("price") + return float(price) + + def _pnl_pct(self, pos, mark: float) -> float: + entry = self._entry_price(pos) + if not entry or entry != entry: + return 0.0 + return mark / entry - 1.0 + + def _is_tradable(self, code, start, end, direction) -> bool: + try: + return self.trade_exchange.is_stock_tradable( + stock_id=code, start_time=start, end_time=end, direction=direction + ) + except TypeError: # some exchanges take no direction kwarg + return self.trade_exchange.is_stock_tradable(stock_id=code, start_time=start, end_time=end) + + # ------------------------------------------------------------ decision + def generate_trade_decision(self, execute_result=None): + trade_step = self.trade_calendar.get_trade_step() + trade_start, trade_end = self.trade_calendar.get_step_time(trade_step) + pred_start, pred_end = self.trade_calendar.get_step_time(trade_step, shift=1) + pred_score = self.signal.get_signal(start_time=pred_start, end_time=pred_end) + if isinstance(pred_score, pd.DataFrame): + pred_score = pred_score.iloc[:, 0] + if pred_score is None or len(pred_score) == 0: + return TradeDecisionWO([], self) + + pct = self._pct_rank(pred_score) + time_per_step = self.trade_calendar.get_freq() + current_temp = __import__("copy").deepcopy(self.trade_position) + + holdings = {} + for code in current_temp.get_stock_list(): + if abs(current_temp.get_stock_amount(code)) > 1e-6: + holdings[code] = current_temp + + # ---- optimal stopping: close held positions ----------------------- + sell_orders: List[Order] = [] + closed_today = set() + kept = {} + for code, pos in holdings.items(): + held = current_temp.get_stock_count(code, bar=time_per_step) + mark = self.trade_exchange.get_deal_price( + stock_id=code, start_time=trade_start, end_time=trade_end, direction=Order.SELL + ) + if mark is None or mark != mark: + continue + rank = pct.get(code, 0.0) + stop_pnl = held >= self.min_hold_days and self.sl < 0 and self._pnl_pct(pos, mark) <= self.sl + if held >= self.max_hold_days or rank < self.exit_pct or stop_pnl: + amt = abs(current_temp.get_stock_amount(code)) + o = Order(stock_id=code, amount=amt, start_time=trade_start, + end_time=trade_end, direction=Order.SELL) + if self.trade_exchange.check_order(o): + sell_orders.append(o) + self.trade_exchange.deal_order(o, position=current_temp) + closed_today.add(code) + else: + kept[code] = mark + + # ---- equal-weight control: target notional per name ----------------- + # candidate opens: top-ranked names whose signal pct >= entry_pct + rank_desc = pred_score.sort_values(ascending=False) + held_codes = set(kept) + opens = [] + for sym in rank_desc.index: + if len(opens) >= self.topk: + break + if sym in held_codes: + continue + if pct.get(sym, 0.0) < self.entry_pct: + continue + if not self._is_tradable(sym, trade_start, trade_end, OrderDir.BUY): + continue + opens.append(sym) + + targets = held_codes | set(opens) + if not targets: + return TradeDecisionWO(sell_orders, self) + + # total value (cash + marked positions) -> per-target notional + total_value = current_temp.get_cash() + for code, mark in kept.items(): + total_value += abs(current_temp.get_stock_amount(code)) * mark + + target_notional = total_value * self.risk_degree / max(1, len(targets)) + + # ---- rebalance kept positions toward target weight ------------------ + buy_orders: List[Order] = [] + for code, mark in kept.items(): + cur = abs(current_temp.get_stock_amount(code)) * mark + diff_notional = target_notional - cur + if abs(diff_notional) / target_notional < 0.02: + continue # skip tiny rebalances + amount_delta = diff_notional / mark + direction = Order.BUY if amount_delta > 0 else Order.SELL + o = Order(stock_id=code, amount=abs(amount_delta), start_time=trade_start, + end_time=trade_end, direction=direction) + if self.trade_exchange.check_order(o): + (buy_orders if direction == Order.BUY else sell_orders).append(o) + self.trade_exchange.deal_order(o, position=current_temp) + + # ---- open new positions at target weight ---------------------------- + for sym in opens: + px = self.trade_exchange.get_deal_price( + stock_id=sym, start_time=trade_start, end_time=trade_end, direction=OrderDir.BUY + ) + if px is None or px != px or px <= 0: + continue + amount = target_notional / px + factor = self.trade_exchange.get_factor( + stock_id=sym, start_time=trade_start, end_time=trade_end + ) + amount = self.trade_exchange.round_amount_by_trade_unit(amount, factor) + o = Order(stock_id=sym, amount=amount, start_time=trade_start, + end_time=trade_end, direction=Order.BUY) + if self.trade_exchange.check_order(o): + buy_orders.append(o) + + return TradeDecisionWO(sell_orders + buy_orders, self) diff --git a/code/tac-qlib/tac_qlib/data/__init__.py b/code/tac-qlib/tac_qlib/data/__init__.py new file mode 100644 index 0000000..92e6e90 --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/__init__.py @@ -0,0 +1,25 @@ +from .config import ( + LakeConfig, + BAR_FIELD_MAP, + FREQ_TO_TIMEFRAME, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, + resolve_lake_root, +) +from .providers import ( + LakeCalendarProvider, + LakeInstrumentProvider, + LakeFeatureProvider, +) + +__all__ = [ + "LakeConfig", + "BAR_FIELD_MAP", + "FREQ_TO_TIMEFRAME", + "UNKNOWN_FIELD_NAMES", + "timeframe_for_freq", + "resolve_lake_root", + "LakeCalendarProvider", + "LakeInstrumentProvider", + "LakeFeatureProvider", +] diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c4e6c345fad1978efe8860c0d977d0c02d6f8d9 GIT binary patch literal 522 zcmaKozfQw25XS8!O`4QSKoK(=$^d-m$sf;ItB#;*T6F<8u$igiL#^-g>%a7PCoa4pYr?DaRz@Zlm|dL2Y5oC+7W%P zR%yRq&1rY-iYI3#fk_h6oK^vnG|2(h`y*A{5LrCrY*Q}kKC|w@ipHCbF&{g{=?mc}U=iYnnz2|)Q-2P{~-Gae$``sUnkyRM>EA*m0CY8W9FdW0?uqzmg z4PY$J5+OWH3=lY?bs^n=4q`GyhV=t_B~2ePgpC8nuxY>)HV>Gy|S;syPrJ(e>LwJ2OLVSU&%c<;A9UCxY)yN?VIF45!=W%v2}0i z2Z{;o493>KiLnhI5vl|R+*3M#^Ej)+uzsFlxj+XMVs3EMP&6_e92w^sF&K?dqg*J) z@j|`D(m%?n1%d*_P@};Kt}e##Vh|PMSPJOZ#yOD^#3;{EH3G*`gM%V7RL4a|f)TDk zxDgD6ga#2c1*xd1!9jn!#X`Zew?EK9cL&aQ(Vgdd8QorFcS`jrYJGU@mao`2e$3FQ5*SD?KF0^q$WhQu@fTGu&pHapm7Uf zhCZrBO^0hKUI@~SM{241`uc+YX@ksnBFkEc?Zi4pN5|=dz&u8ehQo-;v(@@I#DusA z%kWz#NNW;e=qD_oU}OX)X-c4AX=azlmYErfPH;Rk!tKZk)vi&cT$__*ttuZs6^g=2 zWns!9Y;;n{^4AufT1eP|J#>{31lSN^Mi60!zec;xkIPmj5{Zh+<`R0~oxH?|qkf%C zb_Rz;nd}Y3Cpqp06q|%%3aJlABjPBeT#yZo3LST# zZoVRjJWQp?Uz3YY_jX;T`!CY{=Pq=e?hRb%lJTgFkIHyR#)o8lLMA3A{YIIL#yIHq zXmDgy)`gz$HH^ z8-|%7*haEpXq1VJaI!_wCG>bCD9ScrT!02xdN{<4`1LXlqD0$RHmLI}n{yK^8`Q}g z0{x$Rki7s!h;RHU5ObKA%S&J%W$7`<-}-!%O_;_LxJYWb6F7fBG-_`wp1|HESVA;w zDWWw;V<_fI9Lk=S}&k!DKQF#6%C&gMXqoo3nl8~-tmYq9*aeJ5f*-qFBNwmqw?vfDbSOC zgRFxUAnRc=Ii7zGdL>(~GNEyeV4}BKB>5$%l^e5fVGgg_QXF$WJr6CLaRaR9UQURHCOAseY&su}itQA4GZqRC z1x1SCN2rsqb_&h%;2>2KiGtM%g|0J0H~atzkYawK5Ws_j2g3{~a@ed>2Mk^WT!XT#mCaYfMu%>2 zyq}6fSHV0m5sJAEvoS7mfT6@uzzftAsB5r4U=OK!7+}5MBok3VHf9Ga>qah@WgxJKEuqKGjfUC_ATOhVTY=zhgv5obz zHi+$PDQkz=0hrqXv6C%hoe;aG2!FY3=>{ovCvza)PVPA`w3z=b1l{C@LUTe+yfyI=oUQRUJ!`cO03@=l-~7kzK85Ojk5Y70pTVZL2!V zg~~BE_|w1usnIcrw?4>PZl_Yg9uje|n=FyQH5#e2p2x;4PJV33*@_d(M1FigBWd%k z5M-^7I#eS7ei~FlytrVb*)$Fnu7ThdmT|i8zH$4F#RE%wKRL8~XlYl9YDqg=9(?cN z)WaL8_TIF!H)ZNohnt%RsF|}GgYcpZH#e`4kbM%ld1rAkS38lj2s_RDyC97Bbx0_E z*fJ7C7TG^GX|U9<%5-Umb zP)-EZY=i(Tf&p`ttU9T*lUgj3oQGD7f2(_N`fseOt>@CM=cLy2sg~|XXH)f8(#|U> z)0HPBfuJ#T0^hg?>R}Vu&aeC2UVyUMN7HxHx)E%eWbsby8m#GQJt&NA@K(XjA`*H{ zWw5%04)zVH?we^t!Z4Nt>RLN_EG6J~niAw#4wL0e3ug-fr%pF=5=PhrH(*JuzxM=)($1I<%hVV@Hs*V{VN@K2DffaE-B+kF6h(YAVz;jg{m}XjCV5*D^$mU_-<; zFYvP@EMuiwUOr}R+Q?v*JPFGzamPnOooNg71Gqvx(-`XNmM3BP*sj%_Hr?@(*qv8l zW=wrh7gy>}Vtz;bi$EwuW%mXcbU33%G?TBYSupn;51y?N1rZMB{40Zi=vWIv#^}+~ zh1ekSkg47>fx3>|K4DG}nQ*M0>gB+T1ndC{s=!xqXHk=*Tm&#^j+fS(VF0)SV0G{m z0Q}%a7&!I8u~&q{9RNpBLFmu~V07dxi3rfH0*L+fa1J9xMka(<+?ee!KM3;S83?{c z=7~oJyN5ptp{xgH08|MXlS!H#j{$TbqTp#5h10u97#|)E-sA*63Pk~48nZI;+Y8hU z(mt|5>8T(a;J^nbbeV{;vJHVKur!LxP&RkMq3<+u;el%ayaakAS&m0%EZGEAgvlUy zAxM^w#{AYT1`N)ZQ(^*Eik0=;O^^k@A6N=tFfnCv)IbFT zcI57%HDwqt)PtLH1jOM*5Y;X0I}_$Cp4tD_ja5fg+EFDrs*}1*kvnPlSMPJl(;3i_ za13=-FZv}{?Mmqb?*mh+A&_Ul`F1?x^3K_3 z?eAT`A6<<=2skAobs$Fw?=8n%Ef9GV%R=s9*Bq!cFHedZO){2a~ zY`%K7EoH0B6qU~7vt>y`##1pLm_PN-K(Y(SxmRajUEH_SvGUTx#LTObrymAYec;30 zf3jQhoc>_r*O9p*iQ zuj?@{wT{M+wEy~BcsYX$NN;{_^LFVCpYI_d+;rqd%TO^96c9UO1VDjAPu#WDdB|UY zn#;JdTnbM-cCDZabP6YWQ_fpZeWMtHoZ(MVO=)LS%G9J-)UFmnl){WZM76f`v@z{$Oqm*g2R(Idb)|0KRl8Pf8?k16u}aP?be6x3 zf=&oF&3O(m?*`F64KQnF3VGai^2iBbK){scfWepnn%nZ^A;L7AH50^G_HL-K82Iv3 zPDpDgPXfv!yRw}VM$8#j&BTo5QnJ-i&76VFr)lHb&U9*d8c($1<88=5WtRbUBNCmA zAP-;8p;I%=N5kNp*|N@%n@<7$ehdE=w9b#CU=js4QGj$Ve-Z*v*Mc|&r$*#cP|gmr zL5zmL<^ZZx&QrgSN)hbvW;|(Av}Lx{c;S*Q%=jayD7Z-?aUOqn{L#G+_fD2gUT{sxNfrq?q&L<&hTub%F%U>4bi5dGa1az>uyjned7b+EZ%gaCL$}rQ)!`o>DT`J28jh z^4nz0^JJ^q;?($}t?`tRC7?|}bXy;mRO;vpHmh%I8R)N`)uVEn^u{WlETi>h+VD(X z3X1)Wcp2f0u0QIzd(+l$}9 zEV5QKmwPZVXD0Syybmwj@5*Nfo8_g+YMOi*o5Alqp2TkIUdASIzb)Q_F8k040w8*% zM(~5Wex(M^1iVNG1-O1j_gO&ruYv1b^){+{6>fV#w`&GhbN19L-aN(4rD%8s%FD*# zATNkCAmI=wbXrugJ&$1Q*Z&OwuT6R|YMUXvlEh zcum%c*FyrDoILJVVSg53(P##QP3B4!wVH7sY6?FCrI|HXyVJJal8pj$WwqUHz1{kD z`#Pbs?_Kkh&e5~<;$_KmV1|H8C6{}q>CR-vQwslcS7)!zpOrlOmP$UUSguHWj${C* zPs~ot-;hcVE(O!2E#K&HPurK-)KRJQz{kx?6CWL0J#;*M=(u#~WV-a^H+sVJBL0nz z^cF)47ObRnhQHU6DX+YLVBx@G^}@lKj;#J93n;Z3=_Fvj+zAP@8tDCJ`#Fsjjj?UCc#f`}hbXCwj+r8-i zu>6DarH-FetrVrJn^&vb($#HJ^)adB_-aWYT@sK=I+9)4e+B2@&l(Q~n zs{60+>adbs-(t8rBXEj&=lkyv>pFOUEFkFh6VKUV>{0Qqv(36kO*n*`U}z2z{54GY zZwo{Dzbeqd|5pf#@jnK)_xA7<<2_(U;CdNpNw729Xdq;nQSLNR(3kDJR#{-1)9SxN z=(-bfK(-HjK4jMC1HK->4|(9e4PwZnCGh%`DPAA44?}=I>%$?zhog&M=r8 zBk=hFT;XXd2~;l9s|Yk_KLb%!>)@bv6x7tx;9$L?ztlSv8=nMuZqbY?c1B82wu(_2 zZu~gZ5I|&6Hb>AmKcQfpldb4poK~(j!CwI%FTlM$zW}xT`aG^yb(tRV5M=zOfTW?Q zE>&x}6)dgwQE5tu?=EoNZj=9az(QDo;3+!nPWxJsd#-A>YOzQv+LJt$aeC&9@0Tx> zr=5E;?uvQ-etaRGcKb7B74xU&Z-Rp9{N()d`E+GVx~z5P)S9pSzGK0$$S>{xATIfu zR((g)zN4$Y7t+2LBwt|FcOmV&Ao(t?`mUyZS0&%?&FFv8_`Fow@~LMj@E&pBwqRSl z{JwL=19~v!YRNc@lICB3tHXS)0@^nJ&{EmqBRB0+BohU>qYvNa;Ilng{XlJf&@Bn% zpcDF*2Xb`p>)|j(_QXAT;pK1iPQ=aHrQQh{_pA2^ThHilVa9VY{)r1SHSwRIO@V>{ zya+}67sU_X_~4Dy{^wVM56OSD{>=Jd`cWX|>`9q=l#>bH2PNU=t_PyWcI_h|^tH{k zymf_q#pWrOy#`+K9V*wqenUP(MF7g#?We%(S^05WjIGak;rs-jucXp(Gr}rLPoNpa z@G8u6JyB}Jtx73zJQm`vWx2rZ8mbBWb4WnTQow@0NCpu`zi*u0mOK3RCtro%$>IKIrqJD$|WnaF9Ce2o=l;Wi6 zf=|zY<6)HQzS48P=i*B}*)Oen-~+Bc+1+yy61oEYS9-hXj*Hz_F7)&%pWUc;%8Jw; zpr$%ea0>A;9aksrB>xaz!E$0kD}-Qmar|4tg_HlW7sL1b3N!x_^ZgrkAdMaP73N+y z6zwB2q&azHT?f>cq$#;~T@MsoG3hOkYDBaN)58}&>t;k-RN9JY8>Y7?myc^LaD7}B(gJ`cxmm=Dy(q)J)SIbu*x>8Nwh3MTXU4>{0)7z8mdNt4~ ztZ3%s23CX<>jyErD`RtJtd0y2j-rgsl_~aq<1|$hx9rI`HZT(u@?xdu@Dv(2R@#M# a4TBMO+ehL;Zj literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..020dcdcf288e4832c8cf2386351f78d5ceb4fe13 GIT binary patch literal 13340 zcmcgSYj7LKd3SgdPZ9uM5+$7^N+e`HtheXRL2vOwL7M#p0PYlTRUwhX_HO}Lk{7K+^S7xI{8PNwiD-1`+a-3 z0|-HOl1{r47km5o_Sp8VU^Pt{DBjDJy-cgv1Zy}iXTM|$)^a}1 z0l%vZ)KQ9aUZOY`?|#{s>$RHHx}ny?m+Q54q_!MtD|nAyyJpN7@UuH$=rhqsFT=4C z%M1yLkqE~NBGb!;&+{D9JH|)@JabxLIlir(8DfQJhIxsJvgi30yZ!V4FY*P0hN8p$ zOe8KP$RJT-;~Xn+41pix!vkzQBF6SIJv~^{5{XBo#xNV@u~$f&`anxdOOrhk7bRgh z#>b_V4NM=;O2Y!bqBhXe1GD0EQiPRw5oeDvFgep3iF3J66)Ag9Pdgih{;hz;aTt>; zcB)`I;PZmO4hN%W~pZtUsl zO27g;V6ezbPvnRTbMT`>&{K>g;tU_}kHq;VLU6!cf*BtjMN;+i0F^n<^FyNjLP9tn ziT5+ZakT|j#)sKqAU^I6P@EU~*f7t86LCpk!x9q_8Q4dl2Taf8_M;`1A*fr!$SQK*K&#%BD@IoKuUfzeZ@1E=cTL{Co+MuO!+8u}{$ zunfY-9VClC9f`q~u(2UP1PIF^&Kx@?w%C7#H!eUc4mKW7NGvX2><*X|Q%58$DW)!f zqS#IjVLcmF%xB_I2pE;}_S|{b4uj&!tzNJ2;;~fEzE%hGQ~*svu>?093+QXjg1!a4Rs*b6cl_@kE}fx=Zv(%WQ8V^f3JrzpLg z`{w8o>J$|)bO#KI5s0W5wsKg6#VY1#;sP(s(TWMt5fL);+s0P7h^>HGGh&7Xf?_M8 z*s7!0GBl=?g#aE|iBJnefumDm4P+OoyAF3!dT!5^Lz9Oxj=CAwf}?rC)VvIBRM>!d zF4^Fhtc`)WSFDZJ)+TgF`sxNv+Q9s;Om$FK3}Hh*Kt8vLq9zOl{Vn-^T1hr&kO#}O zy8N$3wQ0lU1`|M>(0iYv3gCVo9@W1O9xzvJ@mygLc{uw8$hA^{9zO;rY=jpiM!Q7} zo+@%xfb-}Q01g>&urWp)Lhi&8R!oG4CP`WXX0m09VTe=A$C1S-W%{jCOt9BHTpJ@y zBfwfHg&q}g7Z`>SaEBGQevd;iktl3{%0~DT`yjhW{nA=K<-hdoT~Ec8(aF&_eVLjA zZ~ErjZg_9jU8|d6udiFIZBExVzgydS+n266kntQ`upPYXa9w$1@{zf4#=mVYJX`VY zC$FBJK0CAih0ts`?cbJhY+o>K|MWv6ReSJL#lKB-z*IN49I@Em-dK0UVGF~jo^l&R zFE>He*WqaKRFDRp!obm>Lzp;olIjEP;*u+9;*6K5AbgL~K?__Ii{d9&xrkwmpFv@a zj7TTFz_T3FCnRFPwDzNDKG>?Ywd5o=a@{_VSSZ%_G9ck2Bf!eZm|j)@-Wt}=HsFa! z!#BO$0fiT6xDW8Frb(kreG^n9?R?Udlv9`}N~YmV=Wdlug>;Bx)Etex(JT&wU{!1( z;NKA`6cXG36ZW>Re-0R@$VdB>GDJTlBodNhiLt_YUV_UV3P)K{4249*3wHh@*?IxD zd3;SN8i5E=XV?Z^#5%}+1FXR2nzW{EHFs>Q?)uiGBC}N)-@{4MH*LZy=&)k_Iyd+Z ztfn+Km@wvVp{(-=(VM3RN6Yke180zp*D22U=VtA;{|PNqgC+REXK{fElWY?I^#SAX z_!TADsLeKEmd%5%qGtHi_D}dO96~Az%Ik!MrXtjYRq|=QVPvIN&i``yO0+pTsB=BQ zXKb=%5btPFl5L!&pMJ%vos5Yxxoi;GbW!3g1@LwOJUd6pR%x}?Z4mddD9QH0bwyS1 zsjW?eH@N=6efC4s@#My9J6Asz*IaoV1 z2nGG>@1Kt=d&REx9Eaw98grZ#*n^{c+>FBVz_^oX)J6J%@e2V|{y-;T;I<7V1PSi> z7%%N5nj+q=Jl{yfqhp{l0v{JatBJses4Au6K8EEuhGk-r_^`w`NBI#xnjZ)HWY2vJ zM!<_dUaqQN2exR8N1-LqUaa9nbT}3lfA!yx1Sj-_N?`6|V!$Vq!>1|)eBBA@cx))j zqvFqVj|xIU5CYJw_@ad>8;E;Ql@%+nt`HXy6bnBJ`j)78hQ?SJb3vcnnqz;8MI@9^ zO#RVBuVP1z8IsuED6d#l)C4?T!djfm%!(mYD&h$t#zrIKykbJpqRJ(Xk4kL7DLjPT ziA1vW4WmV(l1CtQ=U$?WF87k1s;Inbo3<^MH>Jy)GUY8v^Lrluy{c7L2c`!Wt2U>rHqVS@s8UJDzp#mp9#ec-xfmZtc1m>r2jC&V}Yf8RqcMYTKsTva9NDHeYL=rRUq{ zccz*%tM*K_XV)+{gV%zy#_xtwbhiCy%6Q|*&8}-*vlXva&sD!%o2lErSa%>@cOX-D z=$)Fw9~h~%%}Zu#)e-t58&$h5CEgspHhO(LQ?q5t{GP8STU|R<_8*Rld%l{KcY05< zBkQld@#M|3*Urw?XV$d7zTu6g*P7mHyR)q`y{2=a?r7S7H2G-O<4-yXzn?R7D~2&L}S_FkPLV+Tp&of2|n*luNoDJ}!ZWihwDvz`q;SU-vHCUC$NGk!(J4(06H0lHe`2MeH2=k!jQntQz&|5H0=Y3nayyBv%CU-oQN3EoUlm$hY08y{z$cc7j_{#e@9~BQ-K?UH z9Dozwg$xb;%Bj8^(u-$aIr8eUxnr;QzA^aP;KJIrJB}moHEqlIw$F9WJoD}L6#abX z)voET7rJLV)4uHs&h6^J@Eym78F9An)*0*AJKcJCcKuszKRo*O z(SJHV)pqsh^wH;!Up+Z}@?HO?h1SDqU-yEuJL{;JI*qns+Oa;{^zf@)b6vNq-f4pI zZRw`NE9YHkfuNYC z^fu8Vn+2lcMnKKw4B-1V#EcVVGzCh03Acr8dL=)DU2D=1 zN!z`3>yyTxSFK5QX8l!Q1gza~bKu&*;@a)$wcF=AZy*2J+RkbFlzB?LTUnD@_2QE+ zoxOE-ep6=CBbmxQQwFT8d2!oI`)}=^H)b~O$W-o}G5~5R@AHp<&4E&6X2VSHOwGIg zElXw8x})@mcB;HGyLRK0ajNrMj-?8!dhL>*s@`xlF`bx+WvX_(Q?>WDA?f(}n#Sab z_npnZtX`XXW~O=GmELeLQ+;T$`tfx2<3Bo)sSYNOz2{w(+CF2SJvPr}y!$5GVTQGV z51f?Kch^^)GCsd&!C9B}c$0QwECh_*LKo1xJ76JpjnEArd25D<8az!z zXFqo?U?P3T%SpJRphdB$QK9j&E;t6nMhcj^y9Fx2wk_9K%?-cyZZi8+dkMjX;ZILFhC?qu4OPYV5?99A3+y9D&OUO0ePjw+cq`}R z%$H1>CKa@CF3tiiWt^L{!mpjPfz#GO>?kKNERW(touw$I!@dp2KUb#)T8UMJvAZaG zWVEnq8=98ScBzyrfE65EVX+CDW-iNP>CUe(SBALSSUxBysO zdr1{mW1y@)0c@(Zejw66&{_}imDc)jG$Hb>^&{wu!QTrk#pofxjo z`6a+g(=`Zz=mkb_PYUP^5|LKq*sDU@387+ebmAOb-#Alp*9&@l4?%E{U8J&}+SHx} z6t!80dx}HJm3C~(mitvN;rev>`kBCNFjM|WvMsx3@88$FS@Zo>AS4gnIyAra&c>ZT z@&2^#pX+|SHra9I*DTiaZKt5)&S;ElFPV>HjrA5YZU{ zkz$wU%F~lir;en=YezG#`lR9R8gQoH2qlFp!;`~T#wW)YY&+&RK&BF{kT2ol2+>Xh zDdFf+gE^i8^?CvxXwdrrH^eJi#xHo4w4;q~iT%BWz_tUjnKL5B@N2@~ zaiHJ-ANF}?BT$cq^_2Dj6z-4}q>S}}^oQ2}6Z%_A(?4Jy|I(A>VV0_&5Va59Q?QKs zJ_}!2sBg0{^wf&?ggnMz8oLv5cp=tDB7*qn3?gfqmJXU`X&q%-EhM$Quo~6MfbY2dmWfa z$20C-N#i|7`L|Di|Lv}K-HbWy-JEO(^Jpr3_5Aetv}b*`8qBHb(W}o+Kl|cvrn-5t zdRMx7*Sz$`_-o^DZG210?C#1`pI9hAk@Zz)9pzKU-f^ILrH`NCX2Y+ZpF5x4yeC_; zI@Ny7HnV+3y0s@=)3jKFSQRPX93Yc5vZ|QwwLll6f?kseWpq z{3#Xg9e+dG(E#Op$dJ+THxrz%&3HhY!dW}es7yN;=#Vl`?_2b3O8Yjw$Y*>_i@u#{ z-_Cgsy_#?N0pDF+8Q+No=Lz8K_w#aKNh({yorR~Y_>7f`(Gw6D0ghbAz@QzIOuv$U zIbf7@wJQIf0C=;ho5XsBei%T!F(CD#SRTMEhS^!jG+BsypZ8c4$wC*t_0ym5jIS@r zsSCUO6rhR7_%BlL+cw|l7x4I|+MgR+u+^xyI6PA5$esmhWLLAedF-o)fmbR>>9k5A!6|WkQ*xb7XilL63vBn)evy^;EsTT`rUD z=R1JB_4%O8U^%R_C`o9ilQf7y&7zdwatP4iZjmHg5jtQ2U#oy?>$5t54#@h7QUN{n zFwJKLC4jqveAB&wPT6_!VBXVO>aj-wb#*^s!OAdpUz7@3nW856gmvQufi&TYx5~~^ zPd@S<$-zx(-@^QJ;pen~3D;-uk6hTkxo`%xoZks;7r@J&Q`+D{n!Cx93UVvK4{GL! z=MxI#@om6Q;xwusbQSIp+oQq?SrDog})lqgoh>$h(sPwIs4VW%Ff zH}wriqlz)k#&cbE?NF=&Pv%l=ps2-%WAJ>!{vrc8ev zOw>44Vldc^jtdvzSNLPh4nqd7MIqE=8eyZNh<<%de5k*w5V7V|=JC~~m`Sb?gGD(J za<|+!_2~2V)YjCtDR zeE!x6Z^QGZsYSSkeZG$wVfguuHt47x1A759JO;&ZR}LKa>3&N689I6>SN8g?mv|(h z$7xOHITvs}*da+28`M=4C!9feS03Y~fdr>2GUSls6%qahvyv{$B(`9NK*W8Jfy?p( zgVk*N#A`C|Uv`_#^~-LPxpB!!S-c+_4CbB7MkplN_-u-VHi(XhS5egh zQTQkp^?^Eh)1`8cdhCFXSwguvN?kXGu@&2X^;P>p;a}hr8Kx*fwq&Gf`U8WLHho-2 z(Hnn3x&EDM$xto7pu9h)eE&r~k*1z_pW6IulbJRwQ;>b&rtGU1R`1A^?YwANw$OCT xvdu*A_{hDHu3T9C&_@*fe$+)DpnVH#T0f%T_oKsxFVOU1I@S6yMG6F~{{f={`MUrB literal 0 HcmV?d00001 diff --git a/code/tac-qlib/tac_qlib/data/config.py b/code/tac-qlib/tac_qlib/data/config.py new file mode 100644 index 0000000..53c9007 --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/config.py @@ -0,0 +1,202 @@ +"""TradeAC lake configuration helpers. + +The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-lake``): + + $TAC_LAKE_DIR/ + ├── market=US/ + │ └── timeframe=1d/ + │ └── symbol=AAPL.parquet # OHLCV bars: t, date, o, h, l, c, v, n, vw + ├── features/ # indicators, wide format, family tier + │ └── market=US/ + │ └── timeframe=1d/ + │ ├── family=ta/symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ └── family=sp/symbol=AAPL.parquet # t, sp_ou_*, sp_hmm_*, ... + ├── calendar.parquet # trading days per market + ├── coverage.parquet # per (market,timeframe,symbol) loaded windows + └── symbols.parquet # asset master +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict, List, Optional + +import pandas as pd + +#: qlib freq string (Freq.__str__) -> lake timeframe partition name +FREQ_TO_TIMEFRAME: Dict[str, str] = { + "day": "1d", + "1d": "1d", + "min": "1m", + "1min": "1m", + "5min": "5m", + "10min": "10m", + "15min": "15m", + "30min": "30m", + "hour": "1h", + "1hour": "1h", + "2hour": "2h", + "4hour": "4h", + "week": "1w", + "1week": "1w", + "month": "1M", + "1month": "1M", +} + +#: bar-field map: qlib field name (without the leading ``$``) -> lake bar column +BAR_FIELD_MAP: Dict[str, str] = { + "open": "o", + "high": "h", + "low": "l", + "close": "c", + "volume": "v", + "vwap": "vw", + "avg_amount": "vw", # amount / volume +} + +#: fields that qlib core/backtest queries but the lake does not store -> all-NaN +UNKNOWN_FIELD_NAMES = ("factor", "change", "trade_unit", "suspend_flag") + +#: columns in the parquet files that are not features +NON_FEATURE_COLUMNS = ("t", "date", "market", "timeframe", "symbol") + + +def timeframe_for_freq(freq: str) -> str: + """Map a qlib frequency (e.g. ``day``, ``1min``) to a lake timeframe (e.g. ``1d``).""" + f = str(freq).lower() + if f not in FREQ_TO_TIMEFRAME: + raise ValueError( + f"unsupported qlib freq {freq!r}; supported freqs: {sorted(set(FREQ_TO_TIMEFRAME))}" + ) + return FREQ_TO_TIMEFRAME[f] + + +def resolve_lake_root(lake_root: Optional[str] = None) -> Path: + """Resolve the lake root: explicit arg > ``TAC_LAKE_DIR`` (no fallback). + + ``TAC_LAKE_DIR`` is **mandatory** — there is deliberately no default + A missing/empty value raises so a + misconfigured environment never silently points at a wrong directory. + """ + if lake_root is None: + lake_root = os.environ.get("TAC_LAKE_DIR") + if not lake_root: + raise RuntimeError( + "TAC_LAKE_DIR is not set. Point it at the TradeAC lake root, e.g. " + "export TAC_LAKE_DIR=/home/data/lake (docker) or set an absolute " + "path in your local .env." + ) + return Path(str(lake_root)).expanduser().resolve() + + +class LakeConfig: + """Path helpers + cached readers for a (lake_root, market) combination.""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + self.lake_root: Path = resolve_lake_root(lake_root) + self.market: str = (market or "US").upper() + + # ---- paths -------------------------------------------------------------- + def bar_dir(self, timeframe: str) -> Path: + return self.lake_root / f"market={self.market}" / f"timeframe={timeframe}" + + def bar_path(self, timeframe: str, symbol: str) -> Path: + return self.bar_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def features_dir(self, timeframe: str) -> Path: + return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}" + + def features_path(self, timeframe: str, symbol: str) -> Path: + # Legacy flat path (no family tier). Prefer `load_features` which + # resolves the family=ta|sp partition layout. + return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + + def load_features(self, timeframe: str, symbol: str) -> pd.DataFrame: + """All feature columns for a symbol, merging the `family=ta` and + `family=sp` partitions by timestamp. Returns an empty frame when no + feature files exist (legacy flat layout falls back transparently).""" + sym = str(symbol).upper() + frames = [] + for family in ("ta", "sp"): + p = self.features_dir(timeframe) / f"family={family}" / f"symbol={sym}.parquet" + if p.exists(): + frames.append(pd.read_parquet(p)) + if not frames: + flat = self.features_dir(timeframe) / f"symbol={sym}.parquet" + if flat.exists(): + return pd.read_parquet(flat) + return pd.DataFrame() + if len(frames) == 1: + return frames[0] + merged = frames[0] + for extra in frames[1:]: + merged = merged.merge(extra, on="t", how="outer", suffixes=("", "_dup")) + for c in [c for c in merged.columns if c.endswith("_dup")]: + merged = merged.drop(columns=c) + return merged + + def calendar_path(self) -> Path: + return self.lake_root / "calendar.parquet" + + def symbols_path(self) -> Path: + return self.lake_root / "symbols.parquet" + + def coverage_path(self) -> Path: + return self.lake_root / "coverage.parquet" + + # ---- metadata readers ---------------------------------------------------- + def load_symbols(self) -> List[str]: + """All symbols known to the lake (from ``symbols.parquet``).""" + p = self.symbols_path() + if not p.exists(): + return [] + df = pd.read_parquet(p) + if "symbol" not in df.columns: + return [] + return sorted(df["symbol"].astype(str).str.upper().tolist()) + + def symbol_spans(self, symbol: str, timeframe: str) -> List[tuple]: + """Listing span(s) ``[(start_iso, end_iso)]`` for a symbol from coverage.parquet.""" + p = self.coverage_path() + if p.exists(): + try: + df = pd.read_parquet(p) + except Exception: # pragma: no cover - defensive + df = pd.DataFrame() + if len(df): + df = df[ + (df.get("market") == self.market) + & (df.get("timeframe") == timeframe) + & (df.get("symbol") == str(symbol).upper()) + ] + if len(df): + row = df.iloc[0] + first = pd.Timestamp(row["first_t"]).date() + last = pd.Timestamp(row["last_t"]).date() + return [(first.isoformat(), last.isoformat())] + # fallback: derive from the bar file itself + p = self.bar_path(timeframe, symbol) + if p.exists(): + import pyarrow.parquet as pq + + tbl = pq.read_table(p, columns=["t"]) + first = pd.Timestamp(tbl.column("t")[0].as_py()).date() + last = pd.Timestamp(tbl.column("t")[-1].as_py()).date() + return [(first.isoformat(), last.isoformat())] + return [("1970-01-01", "2099-12-31")] + + def load_calendar_dates(self) -> List[pd.Timestamp]: + """Trading days (midnight timestamps) for the market, from ``calendar.parquet``.""" + p = self.calendar_path() + if p.exists(): + df = pd.read_parquet(p) + if "date" in df.columns: + if "market" in df.columns: + df = df[df["market"] == self.market] + dates = pd.to_datetime(df["date"]).dt.normalize().sort_values().unique() + return [pd.Timestamp(x) for x in dates] + return [] + + def __repr__(self) -> str: # pragma: no cover + return f"LakeConfig(lake_root={self.lake_root}, market={self.market})" diff --git a/code/tac-qlib/tac_qlib/data/providers.py b/code/tac-qlib/tac_qlib/data/providers.py new file mode 100644 index 0000000..8d0644f --- /dev/null +++ b/code/tac-qlib/tac_qlib/data/providers.py @@ -0,0 +1,230 @@ +"""qlib data providers backed by the TradeAC parquet lake. + +These providers plug into the standard qlib mechanism: ``qlib.init(calendar_provider=..., +instrument_provider=..., feature_provider=...)`` instantiates them and binds them to the +``Cal`` / ``Inst`` / ``FeatureD`` wrappers (see ``qlib.data.data.register_all_wrappers``). +The rest of qlib (``LocalDatasetProvider`` expression engine, backtest ``Exchange``) keeps +working unchanged because the interface contract is identical to the file-based providers: + +- ``feature()`` returns a ``pd.Series`` indexed by the **calendar position** range + ``[start_index, end_index]`` (matching ``FileFeatureStorage.__getitem__`` semantics). +- ``list_instruments()`` returns ``{symbol: [(start, end), ...]}``. +- ``load_calendar()`` returns a list of ``pd.Timestamp`` trading days. +""" + +from __future__ import annotations + +import bisect +from typing import Dict, List, Optional, Union + +import numpy as np +import pandas as pd + +from qlib.data.data import CalendarProvider, FeatureProvider, InstrumentProvider +from qlib.log import get_module_logger + +from .config import ( + BAR_FIELD_MAP, + LakeConfig, + UNKNOWN_FIELD_NAMES, + timeframe_for_freq, +) + +logger = get_module_logger("tac_qlib.data.providers") + + +def _day_freq(freq: str) -> bool: + return str(freq).lower() in ("day", "1d") + + +def _calendar_keys(cal: List[pd.Timestamp], freq: str) -> pd.Index: + """Convert calendar timestamps into the same key space as the lake parquet.""" + if _day_freq(freq): + return pd.Index([pd.Timestamp(x).date() for x in cal]) + return pd.Index([pd.Timestamp(x) for x in cal]) + + +class LakeCalendarProvider(CalendarProvider): + """Trading calendar read from ``/calendar.parquet`` (fallback: derived from bars).""" + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + + def load_calendar(self, freq, future): + timeframe = timeframe_for_freq(freq) + if not _day_freq(freq): + raise NotImplementedError( + f"freq={freq!r} (timeframe={timeframe}) is not supported yet: the lake calendar " + f"only covers daily sessions; add a minute-level calendar to `calendar.parquet`" + ) + + dates = self.cfg.load_calendar_dates() + if not dates: + # Fallback: derive the trading-day set from the persisted bar files. + bar_dir = self.cfg.bar_dir(timeframe) + if bar_dir.exists(): + import pyarrow.parquet as pq + + cal: Dict[pd.Timestamp, None] = {} + for p in sorted(bar_dir.glob("symbol=*.parquet")): + tbl = pq.read_table(p, columns=["t"]) + for v in tbl.column("t"): + cal[pd.Timestamp(v.as_py()).normalize()] = None + dates = sorted(cal.keys()) + if not dates: + return [] + + if future: + # append the next calendar day so that "today" is a valid trade date + last = dates[-1] + dates = dates + [pd.Timestamp(last) + pd.Timedelta(days=1)] + return dates + + +class LakeInstrumentProvider(InstrumentProvider): + """Instruments from ``/symbols.parquet`` with listing spans from ``coverage.parquet``.""" + + def __init__( + self, + lake_root: Optional[str] = None, + market: str = "US", + markets: Optional[Dict[str, list]] = None, + ): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + #: optional named pools, e.g. ``{"sp500": ["AAPL", "MSFT"], "etf": ["SPY"]}``. + #: ``all`` / any unregistered name resolves to every symbol in the lake. + self.markets: Dict[str, list] = markets or {} + + def _resolve_symbols(self, market: Union[str, list]) -> List[str]: + if isinstance(market, (list, tuple, pd.Index, np.ndarray)): + return [str(s).upper() for s in market] + if isinstance(market, str) and "," in market: + return [s.strip().upper() for s in market.split(",") if s.strip()] + if market in self.markets: + return [str(s).upper() for s in self.markets[market]] + return self.cfg.load_symbols() + + def list_instruments(self, instruments, start_time=None, end_time=None, freq="day", as_list=False): + market = instruments["market"] + timeframe = timeframe_for_freq(freq) + + symbols = self._resolve_symbols(market) + if not symbols: + if as_list: + return [] + return {} + + # clip listing spans to the queried window (mirror of LocalInstrumentProvider) + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + start_time = pd.Timestamp(start_time or cal[0]) + end_time = pd.Timestamp(end_time or cal[-1]) + + out: Dict[str, list] = {} + for symbol in symbols: + spans = [] + for begin, end in self.cfg.symbol_spans(symbol, timeframe): + lo = max(start_time, pd.Timestamp(begin)) + hi = min(end_time, pd.Timestamp(end)) + if lo <= hi: + spans.append((lo, hi)) + if spans: + out[symbol] = spans + + filter_pipe = instruments.get("filter_pipe") or [] + for filter_config in filter_pipe: + from qlib.data import filter as F # pylint: disable=C0415 + + filter_t = getattr(F, filter_config["filter_type"]).from_config(filter_config) + out = filter_t(out, start_time, end_time, freq) + + if as_list: + return list(out) + return out + + +class LakeFeatureProvider(FeatureProvider): + """Feature data from the lake parquet (OHLCV bars + pre-computed ta-lib features). + + Field routing: + - ``$open/$high/$low/$close/$volume/$vwap`` -> bar parquet columns + - ``$amount`` (= v*vw), ``$avg_amount`` (= vw) -> derived from bar parquet + - ``$factor/$change/...`` -> all-NaN (not stored) + - anything else -> a ta-lib column in the features parquet + """ + + def __init__(self, lake_root: Optional[str] = None, market: str = "US"): + super().__init__() + self.cfg = LakeConfig(lake_root, market) + self._bar_cache: Dict[tuple, pd.DataFrame] = {} + self._feature_cache: Dict[tuple, pd.DataFrame] = {} + + # ------------------------------------------------------------------ caches + def _load_bar_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._bar_cache: + p = self.cfg.bar_path(timeframe, instrument) + self._bar_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + return self._bar_cache[key] + + def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame: + key = (instrument, timeframe) + if key not in self._feature_cache: + self._feature_cache[key] = self.cfg.load_features(timeframe, instrument) + return self._feature_cache[key] + + @staticmethod + def _keys(df: pd.DataFrame, freq: str) -> pd.Index: + ts = pd.to_datetime(df["t"]) + return ts.dt.date if _day_freq(freq) else ts + + # ------------------------------------------------------------------ fields + def _extract(self, instrument: str, field: str, timeframe: str, freq: str) -> Optional[pd.Series]: + """Return the field as a Series keyed by date/timestamp (None if not present in the lake).""" + bar = self._load_bar_df(instrument, timeframe) + + if field in BAR_FIELD_MAP: + col = BAR_FIELD_MAP[field] + if col in bar.columns: + return bar[col].astype(float).set_axis(self._keys(bar, freq)) + return None + if field == "amount": + if "v" in bar.columns and "vw" in bar.columns: + return (bar["v"] * bar["vw"]).astype(float).set_axis(self._keys(bar, freq)) + return None + if field in UNKNOWN_FIELD_NAMES: + return None + + feat = self._load_feature_df(instrument, timeframe) + if field in feat.columns: + return feat[field].astype(float).set_axis(self._keys(feat, freq)) + return None + + # ------------------------------------------------------------------ api + def _get_calendar(self, freq: str) -> List[pd.Timestamp]: + from qlib.data.data import Cal # pylint: disable=C0415 + + cal = Cal.calendar(freq=freq) + return list(cal) + + def feature(self, instrument, field, start_index, end_index, freq): + field = str(field)[1:] + timeframe = timeframe_for_freq(freq) + + cal = self._get_calendar(freq) + n = len(cal) + lo = max(0, int(start_index)) + hi = min(n - 1, int(end_index)) + if lo > hi: + return pd.Series(dtype=np.float32) + + keys = _calendar_keys(cal[lo : hi + 1], freq) + ser = self._extract(str(instrument).upper(), field, timeframe, freq) + if ser is None: + vals = np.full(len(keys), np.nan, dtype=np.float64) + else: + vals = ser.reindex(keys).to_numpy(dtype=np.float64) + return pd.Series(vals, index=pd.RangeIndex(lo, hi + 1))