From 63c1ea763e88efa6980b82329bc712414657987f Mon Sep 17 00:00:00 2001 From: zhaoli Date: Tue, 18 Aug 2026 06:38:28 +0000 Subject: [PATCH] start experiment 22 (exp/22-re-run-experiment-16s-5-day-rankic-ensem) --- code/MANIFEST.txt | 14 ++---- .../tac-qlib/tac_qlib/contrib/data/handler.py | 42 +++++++++++++----- .../data/__pycache__/__init__.cpython-312.pyc | Bin 522 -> 0 bytes .../data/__pycache__/config.cpython-312.pyc | Bin 9752 -> 0 bytes .../__pycache__/providers.cpython-312.pyc | Bin 13488 -> 0 bytes code/tac-qlib/tac_qlib/data/config.py | 31 ++++++++++++- code/tac-qlib/tac_qlib/data/providers.py | 3 +- 7 files changed, 64 insertions(+), 26 deletions(-) delete mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc delete mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc delete mode 100644 code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc diff --git a/code/MANIFEST.txt b/code/MANIFEST.txt index bbcf811..5634366 100644 --- a/code/MANIFEST.txt +++ b/code/MANIFEST.txt @@ -1,22 +1,16 @@ # TradeAC custom-qlib-code snapshot (auto-generated) -# parent repo HEAD : HEAD -unknown -# parent repo date : unknown -# copied paths: +# parent repo HEAD : 2c2684b103198ce529e71c078a9a93e04f6b113d # tac-qlib/tac_qlib/contrib # tac-qlib/tac_qlib/data # per-file hashes (git hash-object): 1b6298c4a5652f2e863cbdc385a1014a570fcd59 tac-qlib/tac_qlib/contrib/__init__.py c76a9f17f680e74eea766eff27f7624359749ed6 tac-qlib/tac_qlib/contrib/data/__init__.py - 871ff1e163c29261f140c3f53d42a41e6504c779 tac-qlib/tac_qlib/contrib/data/handler.py + 0dd25ef161c6e0f15eafc84886e7e1381deb38c3 tac-qlib/tac_qlib/contrib/data/handler.py b151d139a0dcde87d74b21e7c4b729176ba5c39b tac-qlib/tac_qlib/contrib/model/__init__.py d3f051f3a8650c42fedc7b367b966f7c74fb5789 tac-qlib/tac_qlib/contrib/model/rank_ensemble.py ccfe7d554989aa7f3e5a2128ae663e51b2207149 tac-qlib/tac_qlib/contrib/model/rank_gbdt.py 4afcf9058231111c412925f4c4b84e81d656db87 tac-qlib/tac_qlib/contrib/strategy/__init__.py 79aaad9e39fcc740a773f4f63c512ce1086cfde0 tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py 92e6e90eb0cd0a25142034560f27adb6b705b1a8 tac-qlib/tac_qlib/data/__init__.py - 6d5b9cca6970261fed1f633f6b588ec3e2b399bb tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc - ba4f5d1741728ad1e364e537c7b50488e9112b9f tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc - 608c88f0f45378b170bddb5b301cb32fad9abb35 tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc - 686d36f6d101c547491ca866aa143aa542e17518 tac-qlib/tac_qlib/data/config.py - d9f839be30026f337754a3f015425a8efdbe8e2a tac-qlib/tac_qlib/data/providers.py + 53c9007a928841fd3c3b08450f9a6520ce1ac091 tac-qlib/tac_qlib/data/config.py + 8d0644f6f0d1efb94798ed444cc73e63b643459b tac-qlib/tac_qlib/data/providers.py diff --git a/code/tac-qlib/tac_qlib/contrib/data/handler.py b/code/tac-qlib/tac_qlib/contrib/data/handler.py index 871ff1e..0dd25ef 100644 --- a/code/tac-qlib/tac_qlib/contrib/data/handler.py +++ b/code/tac-qlib/tac_qlib/contrib/data/handler.py @@ -64,9 +64,13 @@ def check_transform_proc(proc_l, fit_start_time, fit_end_time): def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]: - """Discover ta-lib columns present in *every* features parquet file of the lake. + """Discover feature columns present in *every* feature file of the lake. - Returns sorted field names (without the ``$`` prefix). Empty if no features are persisted. + Walks the `family=ta|sp` partition layout (plus any legacy flat files). + TA and SP columns are disjoint by construction, so the common set is + computed per family (columns shared by all symbol files of that family), + then the per-family results are unioned. Returns sorted field names + (without the ``$`` prefix). Empty if no features are persisted. """ cfg = LakeConfig(lake_root, market) feat_dir = cfg.features_dir(timeframe) @@ -74,16 +78,30 @@ def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> Li return [] import pyarrow.parquet as pq - common = None - for p in sorted(feat_dir.glob("symbol=*.parquet")): - try: - cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) - except Exception: # pragma: no cover - skip unreadable files - continue - common = cols if common is None else (common & cols) - if not common: - break - return sorted(common) if common else [] + def _family_common(fam_dir: Path) -> set: + common = None + for p in sorted(fam_dir.glob("symbol=*.parquet")): + try: + cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) + except Exception: # pragma: no cover - skip unreadable files + continue + common = cols if common is None else (common & cols) + if not common: + break + return common or set() + + common: set = set() + # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet + for fam in ("ta", "sp"): + fam_dir = feat_dir / f"family={fam}" + if fam_dir.is_dir(): + common |= _family_common(fam_dir) + # legacy flat: features/market=*/timeframe=*/symbol=*.parquet + if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists(): + pass # family layout already covered + else: + common |= _family_common(feat_dir) + return sorted(common) class DropAllNaN(processor_module.Processor): diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 6d5b9cca6970261fed1f633f6b588ec3e2b399bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 522 zcmaKozfQw25XS8!O`4QSKoK(=$^d-`@7Jp9jb*!yM6?n1L!nc^+vnas;kOLv=<$mM$@WDYsYFaPXy~pDj4)K z2|dZK)PiJ#j9)Y0+8$(<<)N*XCT~&B(wNFanO!F_lWN(h&2*5w S2V3t*<6|x;S+|~?*F`@H0Em15 diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc deleted file mode 100644 index ba4f5d1741728ad1e364e537c7b50488e9112b9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9752 zcmc&aZA={3b~CfH-wO*YA8S6$_!HLV!+>o*{E=Y5v13?&VEZ*B&M@8?VDWzC&H!fL zI(47&1FMb$x*K zvkS(~Q(vTx&An&tJ@?#u&pG#;bI>yTAzvqo%D`;a~A7;;3NL(VLoYseiYcpGnj zi{$NE>EN9JyR_2HdjKxr3!(J#MNk&=B~beKQYg!`vYf8~*w62RvXZZYvRW%^_}u{4 z;&RBtGeQBs=Oiqh-^=gg_wxt%gM1zQtAEEnRH*d^wBAEQUjFb<5r2elc!M4)=3Dqy zzVQv~Pzgo6K=4g(5Pb88l+J-6-?SyrHt~CSy^1(q=GC8kp?Ezz)UtUF({|Dt!zZ#qzNR5%{rl-%6*K)HAccW znQ)8`hd5x4*uYGMd4U;;qbMh3DQ?};jyuNGbNhMLuD4=7W#_Q421bl>>@mG+J<`A+ zG0e6cZD5+3nzrpU#6^S{&!O$S%iAlfjTqrksukpZpAN^U!;_@sciQQ?`S@aTG- z4Grd28}zLNm`EIyhzB;t`1q8VrElnijwC*VJV=}qMbOnKCrScxeTxr7fK=UFEEbnE zy%&SZ2?V3Z07C<esJqhl&PB?z|wZ3@sd&>D@$ zq%mk2P;D(5oGir4d|8wb2vkBh)Z(*O`Yy9qFS1w95A>bA(ml|pl5v$BQ^|-*hE#G= zr6#8Wc9o7N1X%W1cyvs)MB*?-C=wS1)ixQAOhg5ho}A(ms*9T(Ww~g4A||N?XS%Pj zXV3Tb_p$@smjbkE8{tBr+Nv!y#>GYj)ukPCb|My*RF60z!UQ}!65&PzR+R)+;uEIY zbor`IQ^2ZC7hVWVv-x0CZXpPA^REGzCnU2kMSPfr$Dx1w^I^8(4w)h)+UT7mQJrKr z-gq)ayiM_xCss%JawSq7Og!aO!RM&MbG9mOKgry{EWB^H!0+dSQVJbNyV#mNRj}J6+ zi9|Ct-FO}huo?hDxe8-yQCm-amIf@ZR$( zj$L=VGj_*si}uL&J?n-3OyT~et4rt74bT1B_G|IC7hX|bxS>4%YWg`g?HS&*P+r#t zK{;JN*a%O}!e!ZAo2F|wfhw^@IjqnC!Y4QX8Gv~rMcl2GFp~C+;|*~On+Y4}^A$miM^`p_Zf z>$?x}MBuQYix)uLV45W%GR*)vUKrsfBD#m_W};zH1moT;L=)09Gl{i{L0niAL`DSl z)g}ZSBYFTW6=IWN6puxPn8d^cF#3!b1}`E3^+Y@zlOT+M?J!e-13n{+1Z+a4bFeL`_MbcQdC6vLeYhbif2mAx2XY%QsY95P6g=S5ywZm7> z$3wRS6ky`8RB#VmjNxvAG$teg!W(G}d^j@=<{JDD_(P@%HrNz!s8n23?b*$$mQevr z-W9Nt5SD1i4PirqPC%i$U=_gq2~j|*9pVNo=oM0R;@MfeSP>gE2A#pk#aRT_>_r!$ z{YCf_uR?K`*eE0%#c8{Ly|jG(#km*fgLA>9(@JUUjPpx(>3U)121OOrZCYr2lxR3g zF2Ymv&@y}etwP0JyXHP5yALVuhBfz5*?m-TA5YWAQ7;eieolv`=c0ujtOL;fc z9ted!P#5s!d;!#jd<9y9q zLe-lLYsf|*0SIcNH^VW_wKoMr*-dFX!!Z1jo7c4s=D8QsDx?TD=X9vsq^R8zcCsYK zNrw^U9Mi-K6;!Efg8^C-APT}nsuhem*yeVCafBfXkrAy!YZ38aIKa9hI9N6ej+AAS zz8ou>j7Anv#l2A6CDv~puQz_Dbu^9_Kwr2y_$1c-HoLZrB`vVwr8+AaD^*Z`i z?GS^rU<#5Yc{|Ov@ff)tin~O{>wEC}z1NrOmiK>rc;)bNO`2(!z3rdA_}TPlx6)l# zWbc)<P)Elqq+fl+4j7(-S;P?r#G#-nC$+3=%7rVA7rq zzLO3k91U2|4VXx?2k06=E0!EDP$x8O6F%NOHa=T~Z#0f#{6;8ro}})j7S1c)gUjuI z*S0$OwX4)O4UCPY3clu#Sq&P~;TBC3_DoPpwM7`uQh6UyY_4du4C$ z7nSL&H`0Tzre9!XFPnC-|Ihk>@JR*nV?EA^9}UCnR*@H&o7Lp*--!*lCyGa!d|2}` zvX@!frFajo+W*P&>DhmDuXUW4JI*T|7t-zhUz|%fU6Z}n(vE97AM!OHn*`8wa+3kz z+2V%-P6d-i+wX&D07kBmnkwfzu>u28*WvuVZ0SkB@XZn^Oqr&cR@vK{cC`Kk%k^%% z6KbnPV=!I*T0L1TeX04{m){}cQQKD9B70lXj+UQbrA47_x8t3cT412Wk%m-i_rrki*v968T%Ud87;Zm=rd?yjZb!VM+hBh7%s-4xJ!KH zDO&T?$etR-Q#(UuyhRUg-@AQ3nemm)oL?{SK49;$3zwCG+8HY2_ARtN>RRlQ-3Rn; zZ`th41^--Rtvn!?2R_=TlsB!Fx60+MYvmnsd52QowN~COmv<}WJ&LzC?da7t(DbLU z0IVaL)rru5fCgGpmS<_;EQwssAJagKA=5jwWZu(K=4Bpzo1#(4Jis^CE>I^1=C!eWNKyZ%#4?9n-X3PO*QD^bMlm(rC`qTCn@l>HpZHF za{W(YrL4Ra;~Cr=uX+;4Sj!#TkJ7UJ!L;lt+jz~>=Stb1)jL9sl7l#;t$mV03PM&R z4}RxIGz)#s)4V-W6fM5vAc>S?{J;+MEWBfs{K#q2bp8kzoa1#*QsEs2hjqD(2cE<+ z_-tVQQL^~4Rk*OE{^(Ww1XASPSeE+;$@Cff3FIg9&%5jCyt+fbM9h-+A5RgtEiVyM zWWbXQ;$$w)Rf}*TuNMPgyI-ysCHS7fhDFHy<9sCmfg6xo(QlQyU4?8O7<$9|8bN)N zc5BnFGn%k&XfJ8Wi`1bekfMapBbhaO9JkkRHldjey zot=#>t&MFhfie^WEQI4C^cpVk{abbR-3|#_0$O$ULOSj&{_X)uTOo$dVXwBRJ-AnT$+oLxSJ8k*6I#QwQ?ex zm=OYMWPSymfGQuc;R4W<W0x)~Wp%R@BoB*x zv#s~1GG!I;fByR1^@Vdv*@5NKkE>RyPK~pbxV5| z56$+hSMPq*zu3R*TYhO}w^H4)R((pYKBZJYH`}*fRrBcN;>l&p^0}1)rRw-v)d{)k zgi>{Cwl`C=_Yu3uDm6{3^sfp(DO8R;pRVpsSDi_ho&i_=VBp@sLSX5-;%P8 zp8Bs!O6MEr8kflBp5(&Ob<>8_0RP$`97$8uX4HP{o2)Hc~9Hgo=$mBr?TgS zQhIW&v|BFiR!Vzj`qq7A^C#y{E?GWsz2{nPd*8cC$xPcC(K z*E7?bDXYXBR>}@8(;pYE6e{%{>C(=$uPg28$`m!Ey^U!{{Egopt_88HwY2AOqc=lNl1onZw!~;46(RYZt zQK7{VxoCce$RQUs%~AJv(J<-`JxJhNKnSv9_-zU(p<$MWhnqC}rRU5%C=9&YW*;@ns`aaGDb7M>SHKfNs4S_@ z82;WG2`2^BjlX}eS_)fKZSau_zR#dP19AbYZq;(O%!+A86)iHZ!-Cy~raItU+EgyU@srXyh{KsVfu{Hm5 zvi~{7-@WD^ko^OS|Kghey6nHM`2S?q`en;;rK0_lvgPh~s7Ibf&(h_0y{lzl2h&CE z8E^57^MAg#5dIDkADe$^ukP{FTh1xwi59`*-yY1vFDF3z0owlHtQhp?{l4J%EKj!*grN zNgNjFd8!c%X(tWeU-g*eHiq8B#b4rL5sDQ(!Jr)tbQ!neLd);H0Kgl>mvrz4nzA0< zbdlCUa??gyFKgwnR$kZ2Hr(8bi?&UN!`eq~mR4JvHcz^&B|rE{>qSx*5T*?K_+?!){DS8AAm&Xiq1=U?HsA z2u!pdEX0B}%V7O3+y%;Xi;bIJz==eSim*V!T8)X7CaH}>L_tx;Eu7zw4K=hwBNUf zJAe>mPugk!>PlSf?ep7jcYohw_g$CEPC@w2zh4}DtDd5Mfgf5hs|kAPpCNIL;;10S z(VQVl#|%LOjcH@l7&OAKDQb$DgXWkeXo*>a)|f45%axY}?WCR~=!9Q$)D?3F-7!zl z6DtpvlRB1YMa&!YLfXpNqQ00v=#NzfD`Qo`s#tZfnx+gC@8!x~qE%{wHJq2TUp576 zIUnbM-xUVxFvU49Q=E%;zhunST1j%g_1rp=TMoGuyhqPnHD(O>*=^ADd^FO_ zaIC~KLqcLC!tsL0^s?b|Jje8oG136foDx`$Z)<0USmFFIFELT}9N%KMpBmsrzM#=i zbhw|1#H9pjBuZ?YV+D>O{m1z502_~pvE58h59YK);t{Da%tm>v71D;@*V59`WRJu} zNf?gtacOx0)5o*Yu)r_N4fOQDs5qPyVI^L~(PIn@&h$p&T<%jP%HGq{&PJhrD^PJ1 z+T@a*YPTKmc|l-@hL8n~BF}5IB6nIyB=G$aQR0OV8;ypvx;;IC7UTva@S?;d`qU{m z_VjcmV1gacSmdQAa?FJ>_|YM#DMk`;hL870;(QZfIM6P^h>wmUtNMB9l{v@rL!$jc zLO2(R_cOzBwFG9yhuL9ZKCTXMoEQ4oFwcY&aYdY9s`Zp#x0}?5eX9Jw3TT5-bWG9T z#nc78D7NE6n9oKP^XWJw0!F30J-6Sr&7gR4v)41cxGmMaujRow6~L2FEWr&&`A{^` z-_HxbLTG>yU5DCEggTEt+SL&{*7k&A?}D@0o{0BF`W4^l?#H{2f1^8B+}(EU(UXc- zf=TrWY>W@}C4^9)z@Hzlme_EJ97?jshcp-n=irCH3xW*%(Myj&;uLj)djVWaz=#R`qsD-A0=#*Fk(Ix7R!=03#*>&~6T+%2X)mDHuOW!XE#vPgrT6mC0|cV$p)?G z!7?o`|EqCr+Hj@81U*jZwNFw7{eBf4)xQlLFjj5xSYZ%(IQj*MwN!u}KLR^!gcl@6 zJ4FocDsof+d2|T?n+yP!k^d&(Ix7a*77O;<%@Sb6<0?mN8j{i zYxceAn{T_}y;*m?Zian+%|dN+rndS0+SWIGnVNlB&;GP+{~d?x>LZhn%!RZ5Epy@7 zitnDdc4qp_%-ZKdv*C6FH=GI9;Jg8I4BmyPmXdCV2qze zVT?eevtHm?j_DH;F#xT-D4O@TYGo}siH)e+2NDa#`fdg!d}IWmoV4j>1;Ey@zPEu+ zBpSZyr4C3uN5gr5Uo}h`W$HViB5C`Rrlg$0L{TygdpdWjWGbZhI7ZFUh>d1(7zC?g z3juydq)*xZR1xbS`VByV%{6Jw*lKRuR^0KeN=0U?vc89trtjE<6;NT>{B<<= z9?YgR8cZ1Tr%={0LiFaj!O=4Pyn!>w#^)){_-AJAwEsRWQ-dY&;LA{8!X%r-e|kVW z+ztEd=0we=Ie51WvRg7P|Hp{WQpVU>JZZD?7krSrd>J`-(>4(etQqbQH=+DkkvQ=8CRU5>0EDExHa7|Gb zd}?#k`WsyP;5vJu=*lA|>PoMPDww%-aQ%bZX#J8^0lYs+$p&G&Y~pMhB+qD|^|l)f|W7ei~yO72v_qJ#I!}xo_ObH0mOK-}r?9Du1ApFmT$25`qNh ze2kZN6HO6MR~|PK@#q++jDX`JXf+Y|5LKmA+{3UO$FNK+5+9cM<|seHNAvAKpX|90 zU<5q)DMc^es{G42`kS=7K)BH3xr+MI@Y1 zO#RVBuVP2U3`uNnlvgY&X#(ypVKt6rX2lRH74d`+W22FAUNNC)QRNcHMx<4?&UEvEEOYQ@wQW;vi!17GHeYX^rRUq{ zx2Kx3D|St_FRo&42CoNajo%NY=*9NKDdUYpH@mKP%~rfpJy-owZMJUfLfyVh-M(zy zf%j?-eq^LpH{UZ;D-O}0*r?hyDe>m$_0i|Yvo)Kh%pdq_7OQKg%Kp_+ao1Oq@=otc zb}ag9Z=AS!=K7i0`s}LK*Vet>^lH=FZMU~{W>$5k>keo9hm(&kdi+Tz!TUKww_-Sb za?T{6VE)3Tsp??12xo<6$P@e!1*|Bp;;1Z#6njY1z(a~N6gof5Msp?BP>4%}L875d z5*~v{aplBTm&L}S_FkPLV+Tpk26LI8=-s9IUC?dT`bf=RE)8z3Wo^AbM4_V4XVPS~&>+(VRt?BvuxdP7^#Ti~Ld(nl zYEl@iidCvr0hS9|b|lEOgq$ZqAMFX$nMNp~OQr{PEkvtDf$L3^h6OZjMw(4*fD~TiW$@l z(CW?NP&5J!06!x`0y!~6ayyBv%0WfIs2(lESA`arn{wqL@QEd&BYY@Vd%WR6C#$F; z2i=M9Lxcu@4noTAA04;+>zIMUmtvRFul6%w&TzTOrOkn7abK-r_ff+IMyyUJ^V`7T-O^_?=?aD zwoKE(<>RKCpix`e*|zBMp>4Y0sn2-ov!0D<+s4m7c2NFppZ}Y$8~)lvR5RXWI%-Tm zG*))3GX1cYf%xxMG9BwoVf~mx9o=mCdjb(M&5_MFVJSFvChZ7KSYaappcDsLSHnS} zw}}?nED#kp0%|U20N=MEW}GOaDNy1|xGiMU%lQ%Pl0)mg6wJs5z-Tk{WakWcNOZ~! zU(pV+U2SJx*3Jm#i$k`7Ir1$41!q4s;et8pFr(1|pjYC!%T8H0QJ{x26m9L6>_;;aSrTLfUw78S4r~&Iv&%Cl1=8v%IgY~xJ-Nl3-%zWL@lKrwzHFdOE4wf*m zWS=&hiAn<{R{||y3d^RKaw2NNaRp!%c9%n1udR%%8%AJKIZ!R>Rs}E$IPeY{+rbTl z@GGEUuQojEQ0Jl&JX#!Ju&|=3cm> zh$}`U(hyB3hJlC>!S=2kzf`Iex>K>`awHMCii*wsK{ZRtk(NTg`vbwbxMB=?7M~{O z$znxa>d8#S#-t^0sND5dEqEI;-iBFA*4vu2e&DHs|3tfA^i?L0erlo08t%F)u0|)L zGp1xT>)wzw-l?o6F1bzbt=lngoNs&8lC9jCwBPYnUfVmpcgFS}vvsz8-Y|DK>)V#J z-CeUbY5aNBs$}P)zY2_i)$48!Tpw6iy*0CX>wM=MM}M}ubJ{*-o)Yg=)}&UvaN@-? zx6aIO$ZmKfTe)k>fSENfYRf~o9UgYdEdY3 zUKzFKF#WNeDz98zy?)9#)%homdlgjm>U(~wdfm0ebYdo!t=jfp)$TV8NypDuH71XJ z=xqLF_3G65ndW&{X5Id5^?`-z$1~NB|KqW2bufA41MiB|)*1Wkk$Eob-80z^BdiX5 zD16UG)i*9ytypZ{e6P%I-f`E*+%v;3=*SiRdsa-@ zDDOj`V%buS)!guF?*_9^wU@BFF#PEu2RQW7(~wncFL7m@y}&+lMTWfI_x{J{d0NhMJut2@a`^( z9vSUgwek3ouJ*6PRUkNI9)e3czB!zT4S@%SVJtD{t@Q(u{(;tdxL;|l4@VOs-&#L{z8L(yz~a3Y3C;U)RGpBVyJmTR zY%DPx#|zWOeay(FkqdCE2BWYe{h?(g;DBPFyJ8=+*7x#0;nosv{eU!b;n@nebQ(n} zU4=1p3Lfm{Znm3gB>pXE$#a2%4%zsaMD7*&C|r$xWxx(K#;P-hs|(^M)7ZR>=w&Wq z+*C%iS9>5)WUu}=T;$8`RPu$UCJCC=YAI>8+C%6C3$x(?9xg+2gsb1eCTK4rrND|p zi?y!A9@Wj&o7m_fh<@`xpZOKr;`W_M(-qI+u02W9`?i(jgBgJ>KV1?|WRm6EQf4(euazA|VeKp}>> zub^fLvsmk~^bBMp!q{5GhXLozy&+WgVcZ zbmDdGoa&bVD^1rR1fmxh!96LUGe|^Uk!`OEZ6{n6gQFA2;QGdylB-@&<1era4x&rc zqNg^sD~+Od(czxrP;zA)8y3s`s+VwWrhM&8U^bX7e-^~WZ=Iepzn%}nR0Lw z&u_|>?}J^jZ~qUTeDle-f-eX!Uc7a2{`~ERT|YVT(=$Il^X}8hjs@qMjC0M5G2^U< zeX-zKm+`Ed*_!b*q-_oA&M1TegfIfOqmU@Pk1GWjGWdy>nT3Y@om~E=PN)KpP@(Y* zJ!2^4EyMdAknM(U;bEu&?m==)9a4haLlBi62UHB4B%qZ97uE)@PgUa$QjwR#Ozx@2 zYfCa8?-^c$9z=AI!0olmbM@)Tr&EVg;`PH>SAEiOXB9a3Z-kP<)#1tEtK*a7Y1_8> zbr7jcE5u5i9AVliU?m_hHJHOW$kzklvQbA;vuwz_62YZ)JpUyV3w^>OgFm3u;|Hj{ zQ#Q*Myy+NNEFU+4zx@m$Qd6<)#IG$ zfo1z-GiOAK;n#$}ZeP~aX7^C|6@C|n^cNIvTW`46T4EBv>X=6}FE{*4pl zv6!ke5mgl4c(9E6W(;3`sPDz_I;<7X33(vGG7Qo#KofOCR^8X{Wwin$L&7(BLu7j_{Q?g51xSOuNb!uKEo z>*OI!PeTNCkw_vn)#GUd*$6KX5>a@01gf=&+mr}Uh*TMB3-P1mP2G5DttnUf*U(UO z!wI`Y-F15x+-oxKHDISI6I8|F)Ff*_Ew6mM%ZG=&N3IlusRb&w(bN z-hPIg4Zm`3?p$W$uEm;_srKu(nXNO@tzDU#riJQlnd)t?oz7P8S*Y&FRCm06>PNwM zg6SttrcZw}`)Dv*{ZzXADYf6*{)UXB0n&F7kkJ`A6P&NjdO*X&Q9IDk%{UmSkTOs2 zS@3Ph_%^)2XMIfzzU>*`_IVCHt#A8*-W^?8-?6mw7$E!oSPqb+!V<15Jg>#4wN#9r zfXf;{Yu0e-`?o0r3!lDI3M~07fy4&OijYWHX>g zHSa~yROH->yYT&?{;X(xZAqjqtdfS_L_xUP3zNz+S#?rPLl^KOX2e%u+ z=${abF7VUp9z9?XwPM+0GFg9s0eDQ6WLdt3wSZw(4Lv7pc}_5jNk?RO z$U%>T2A%g}=5=SizFjVp?dLjxz4iHvqQP>QXHk&QcR<1*UbGg4{F1|^5v~>qvK4y= zD&T7cpte4$1NeZ@R}>2Psb3v^*+mJUE66w9L+O;Am-gqqzoj0a6mVDfT^7s?-xCyt zf>Nfa2tHxnct9XbxZuJyTEK+s%hyLPEZCHNJ752J$+!3$Qn{xXfQQBlOZ22FgZ zzp4mK8!fdr>2GGvqE z5fR?PsH7V;i6t042Y=!oh`^2dk-=)VedaZp_b$23=6Z}8@3|?9_hW;>ynV?Cse2@4 z_nG}mO(t{gXU+=qL0aWE_IBUI6jN^^5e3FFP%8WYqvbOf{uZ09CJtbHE~7q;f_E_p zYKo;dBJ%J$-U3(v?n=Rg9iYS)P$mi}*%dQ6sp@IBz;y=4iUbE#X^!$zKzELSo*czS zZeQRzynx08KIJ0UEksAeqo`_uD0~!)`e>fKGgHx{9xHquBb@6TrOq3#))m`c^`-rO z;a&Jdz!W8j?ip#C{>b2@O`q0L^!i^=u79CgvQ*12DDTfH-+xd~WT+=Tq&EKAWTp*E z6ht4nDf`Ow%5B-Q?UyV|7MgBZvYF^@pSai4mFblaeL}(SCtdVD+LvC{`UwTUpByxt Qpy`8js`XQfBnVai1x>3EVgLXD diff --git a/code/tac-qlib/tac_qlib/data/config.py b/code/tac-qlib/tac_qlib/data/config.py index 686d36f..53c9007 100644 --- a/code/tac-qlib/tac_qlib/data/config.py +++ b/code/tac-qlib/tac_qlib/data/config.py @@ -6,10 +6,11 @@ The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-la ├── market=US/ │ └── timeframe=1d/ │ └── symbol=AAPL.parquet # OHLCV bars: t, date, o, h, l, c, v, n, vw - ├── features/ # ta-lib indicators, wide format + ├── features/ # indicators, wide format, family tier │ └── market=US/ │ └── timeframe=1d/ - │ └── symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ ├── family=ta/symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ └── family=sp/symbol=AAPL.parquet # t, sp_ou_*, sp_hmm_*, ... ├── calendar.parquet # trading days per market ├── coverage.parquet # per (market,timeframe,symbol) loaded windows └── symbols.parquet # asset master @@ -107,8 +108,34 @@ class LakeConfig: return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}" def features_path(self, timeframe: str, symbol: str) -> Path: + # Legacy flat path (no family tier). Prefer `load_features` which + # resolves the family=ta|sp partition layout. return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + def load_features(self, timeframe: str, symbol: str) -> pd.DataFrame: + """All feature columns for a symbol, merging the `family=ta` and + `family=sp` partitions by timestamp. Returns an empty frame when no + feature files exist (legacy flat layout falls back transparently).""" + sym = str(symbol).upper() + frames = [] + for family in ("ta", "sp"): + p = self.features_dir(timeframe) / f"family={family}" / f"symbol={sym}.parquet" + if p.exists(): + frames.append(pd.read_parquet(p)) + if not frames: + flat = self.features_dir(timeframe) / f"symbol={sym}.parquet" + if flat.exists(): + return pd.read_parquet(flat) + return pd.DataFrame() + if len(frames) == 1: + return frames[0] + merged = frames[0] + for extra in frames[1:]: + merged = merged.merge(extra, on="t", how="outer", suffixes=("", "_dup")) + for c in [c for c in merged.columns if c.endswith("_dup")]: + merged = merged.drop(columns=c) + return merged + def calendar_path(self) -> Path: return self.lake_root / "calendar.parquet" diff --git a/code/tac-qlib/tac_qlib/data/providers.py b/code/tac-qlib/tac_qlib/data/providers.py index d9f839b..8d0644f 100644 --- a/code/tac-qlib/tac_qlib/data/providers.py +++ b/code/tac-qlib/tac_qlib/data/providers.py @@ -173,8 +173,7 @@ class LakeFeatureProvider(FeatureProvider): def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame: key = (instrument, timeframe) if key not in self._feature_cache: - p = self.cfg.features_path(timeframe, instrument) - self._feature_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + self._feature_cache[key] = self.cfg.load_features(timeframe, instrument) return self._feature_cache[key] @staticmethod