diff --git a/code/MANIFEST.txt b/code/MANIFEST.txt index bbcf811..5634366 100644 --- a/code/MANIFEST.txt +++ b/code/MANIFEST.txt @@ -1,22 +1,16 @@ # TradeAC custom-qlib-code snapshot (auto-generated) -# parent repo HEAD : HEAD -unknown -# parent repo date : unknown -# copied paths: +# parent repo HEAD : 2c2684b103198ce529e71c078a9a93e04f6b113d # tac-qlib/tac_qlib/contrib # tac-qlib/tac_qlib/data # per-file hashes (git hash-object): 1b6298c4a5652f2e863cbdc385a1014a570fcd59 tac-qlib/tac_qlib/contrib/__init__.py c76a9f17f680e74eea766eff27f7624359749ed6 tac-qlib/tac_qlib/contrib/data/__init__.py - 871ff1e163c29261f140c3f53d42a41e6504c779 tac-qlib/tac_qlib/contrib/data/handler.py + 0dd25ef161c6e0f15eafc84886e7e1381deb38c3 tac-qlib/tac_qlib/contrib/data/handler.py b151d139a0dcde87d74b21e7c4b729176ba5c39b tac-qlib/tac_qlib/contrib/model/__init__.py d3f051f3a8650c42fedc7b367b966f7c74fb5789 tac-qlib/tac_qlib/contrib/model/rank_ensemble.py ccfe7d554989aa7f3e5a2128ae663e51b2207149 tac-qlib/tac_qlib/contrib/model/rank_gbdt.py 4afcf9058231111c412925f4c4b84e81d656db87 tac-qlib/tac_qlib/contrib/strategy/__init__.py 79aaad9e39fcc740a773f4f63c512ce1086cfde0 tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py 92e6e90eb0cd0a25142034560f27adb6b705b1a8 tac-qlib/tac_qlib/data/__init__.py - 6d5b9cca6970261fed1f633f6b588ec3e2b399bb tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc - ba4f5d1741728ad1e364e537c7b50488e9112b9f tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc - 608c88f0f45378b170bddb5b301cb32fad9abb35 tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc - 686d36f6d101c547491ca866aa143aa542e17518 tac-qlib/tac_qlib/data/config.py - d9f839be30026f337754a3f015425a8efdbe8e2a tac-qlib/tac_qlib/data/providers.py + 53c9007a928841fd3c3b08450f9a6520ce1ac091 tac-qlib/tac_qlib/data/config.py + 8d0644f6f0d1efb94798ed444cc73e63b643459b tac-qlib/tac_qlib/data/providers.py diff --git a/code/tac-qlib/tac_qlib/contrib/data/handler.py b/code/tac-qlib/tac_qlib/contrib/data/handler.py index 871ff1e..0dd25ef 100644 --- a/code/tac-qlib/tac_qlib/contrib/data/handler.py +++ b/code/tac-qlib/tac_qlib/contrib/data/handler.py @@ -64,9 +64,13 @@ def check_transform_proc(proc_l, fit_start_time, fit_end_time): def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]: - """Discover ta-lib columns present in *every* features parquet file of the lake. + """Discover feature columns present in *every* feature file of the lake. - Returns sorted field names (without the ``$`` prefix). Empty if no features are persisted. + Walks the `family=ta|sp` partition layout (plus any legacy flat files). + TA and SP columns are disjoint by construction, so the common set is + computed per family (columns shared by all symbol files of that family), + then the per-family results are unioned. Returns sorted field names + (without the ``$`` prefix). Empty if no features are persisted. """ cfg = LakeConfig(lake_root, market) feat_dir = cfg.features_dir(timeframe) @@ -74,16 +78,30 @@ def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> Li return [] import pyarrow.parquet as pq - common = None - for p in sorted(feat_dir.glob("symbol=*.parquet")): - try: - cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) - except Exception: # pragma: no cover - skip unreadable files - continue - common = cols if common is None else (common & cols) - if not common: - break - return sorted(common) if common else [] + def _family_common(fam_dir: Path) -> set: + common = None + for p in sorted(fam_dir.glob("symbol=*.parquet")): + try: + cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS) + except Exception: # pragma: no cover - skip unreadable files + continue + common = cols if common is None else (common & cols) + if not common: + break + return common or set() + + common: set = set() + # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet + for fam in ("ta", "sp"): + fam_dir = feat_dir / f"family={fam}" + if fam_dir.is_dir(): + common |= _family_common(fam_dir) + # legacy flat: features/market=*/timeframe=*/symbol=*.parquet + if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists(): + pass # family layout already covered + else: + common |= _family_common(feat_dir) + return sorted(common) class DropAllNaN(processor_module.Processor): diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 6d5b9cc..0000000 Binary files a/code/tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc deleted file mode 100644 index ba4f5d1..0000000 Binary files a/code/tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc and /dev/null differ diff --git a/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc b/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc deleted file mode 100644 index 608c88f..0000000 Binary files a/code/tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc and /dev/null differ diff --git a/code/tac-qlib/tac_qlib/data/config.py b/code/tac-qlib/tac_qlib/data/config.py index 686d36f..53c9007 100644 --- a/code/tac-qlib/tac_qlib/data/config.py +++ b/code/tac-qlib/tac_qlib/data/config.py @@ -6,10 +6,11 @@ The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-la ├── market=US/ │ └── timeframe=1d/ │ └── symbol=AAPL.parquet # OHLCV bars: t, date, o, h, l, c, v, n, vw - ├── features/ # ta-lib indicators, wide format + ├── features/ # indicators, wide format, family tier │ └── market=US/ │ └── timeframe=1d/ - │ └── symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ ├── family=ta/symbol=AAPL.parquet # t, sma_5, sma_20, rsi_14, ... + │ └── family=sp/symbol=AAPL.parquet # t, sp_ou_*, sp_hmm_*, ... ├── calendar.parquet # trading days per market ├── coverage.parquet # per (market,timeframe,symbol) loaded windows └── symbols.parquet # asset master @@ -107,8 +108,34 @@ class LakeConfig: return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}" def features_path(self, timeframe: str, symbol: str) -> Path: + # Legacy flat path (no family tier). Prefer `load_features` which + # resolves the family=ta|sp partition layout. return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet" + def load_features(self, timeframe: str, symbol: str) -> pd.DataFrame: + """All feature columns for a symbol, merging the `family=ta` and + `family=sp` partitions by timestamp. Returns an empty frame when no + feature files exist (legacy flat layout falls back transparently).""" + sym = str(symbol).upper() + frames = [] + for family in ("ta", "sp"): + p = self.features_dir(timeframe) / f"family={family}" / f"symbol={sym}.parquet" + if p.exists(): + frames.append(pd.read_parquet(p)) + if not frames: + flat = self.features_dir(timeframe) / f"symbol={sym}.parquet" + if flat.exists(): + return pd.read_parquet(flat) + return pd.DataFrame() + if len(frames) == 1: + return frames[0] + merged = frames[0] + for extra in frames[1:]: + merged = merged.merge(extra, on="t", how="outer", suffixes=("", "_dup")) + for c in [c for c in merged.columns if c.endswith("_dup")]: + merged = merged.drop(columns=c) + return merged + def calendar_path(self) -> Path: return self.lake_root / "calendar.parquet" diff --git a/code/tac-qlib/tac_qlib/data/providers.py b/code/tac-qlib/tac_qlib/data/providers.py index d9f839b..8d0644f 100644 --- a/code/tac-qlib/tac_qlib/data/providers.py +++ b/code/tac-qlib/tac_qlib/data/providers.py @@ -173,8 +173,7 @@ class LakeFeatureProvider(FeatureProvider): def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame: key = (instrument, timeframe) if key not in self._feature_cache: - p = self.cfg.features_path(timeframe, instrument) - self._feature_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame() + self._feature_cache[key] = self.cfg.load_features(timeframe, instrument) return self._feature_cache[key] @staticmethod