start experiment 22 (exp/22-re-run-experiment-16s-5-day-rankic-ensem)

code/MANIFEST.txt

@@ -1,22 +1,16 @@
 # TradeAC custom-qlib-code snapshot (auto-generated)
-# parent repo HEAD : HEAD
+# parent repo HEAD : 2c2684b103198ce529e71c078a9a93e04f6b113d
-unknown
-# parent repo date : unknown
-# copied paths:
 #   tac-qlib/tac_qlib/contrib
 #   tac-qlib/tac_qlib/data
 # per-file hashes (git hash-object):
   1b6298c4a5652f2e863cbdc385a1014a570fcd59  tac-qlib/tac_qlib/contrib/__init__.py
   c76a9f17f680e74eea766eff27f7624359749ed6  tac-qlib/tac_qlib/contrib/data/__init__.py
-  871ff1e163c29261f140c3f53d42a41e6504c779  tac-qlib/tac_qlib/contrib/data/handler.py
+  0dd25ef161c6e0f15eafc84886e7e1381deb38c3  tac-qlib/tac_qlib/contrib/data/handler.py
   b151d139a0dcde87d74b21e7c4b729176ba5c39b  tac-qlib/tac_qlib/contrib/model/__init__.py
   d3f051f3a8650c42fedc7b367b966f7c74fb5789  tac-qlib/tac_qlib/contrib/model/rank_ensemble.py
   ccfe7d554989aa7f3e5a2128ae663e51b2207149  tac-qlib/tac_qlib/contrib/model/rank_gbdt.py
   4afcf9058231111c412925f4c4b84e81d656db87  tac-qlib/tac_qlib/contrib/strategy/__init__.py
   79aaad9e39fcc740a773f4f63c512ce1086cfde0  tac-qlib/tac_qlib/contrib/strategy/optimal_stop.py
   92e6e90eb0cd0a25142034560f27adb6b705b1a8  tac-qlib/tac_qlib/data/__init__.py
-  6d5b9cca6970261fed1f633f6b588ec3e2b399bb  tac-qlib/tac_qlib/data/__pycache__/__init__.cpython-312.pyc
+  53c9007a928841fd3c3b08450f9a6520ce1ac091  tac-qlib/tac_qlib/data/config.py
-  ba4f5d1741728ad1e364e537c7b50488e9112b9f  tac-qlib/tac_qlib/data/__pycache__/config.cpython-312.pyc
+  8d0644f6f0d1efb94798ed444cc73e63b643459b  tac-qlib/tac_qlib/data/providers.py
-  608c88f0f45378b170bddb5b301cb32fad9abb35  tac-qlib/tac_qlib/data/__pycache__/providers.cpython-312.pyc
-  686d36f6d101c547491ca866aa143aa542e17518  tac-qlib/tac_qlib/data/config.py
-  d9f839be30026f337754a3f015425a8efdbe8e2a  tac-qlib/tac_qlib/data/providers.py

code/tac-qlib/tac_qlib/contrib/data/handler.py

@@ -64,9 +64,13 @@ def check_transform_proc(proc_l, fit_start_time, fit_end_time):
 
 
 def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]:
-    """Discover ta-lib columns present in *every* features parquet file of the lake.
+    """Discover feature columns present in *every* feature file of the lake.
 
-    Returns sorted field names (without the ``$`` prefix). Empty if no features are persisted.
+    Walks the `family=ta|sp` partition layout (plus any legacy flat files).
+    TA and SP columns are disjoint by construction, so the common set is
+    computed per family (columns shared by all symbol files of that family),
+    then the per-family results are unioned. Returns sorted field names
+    (without the ``$`` prefix). Empty if no features are persisted.
     """
     cfg = LakeConfig(lake_root, market)
     feat_dir = cfg.features_dir(timeframe)
@@ -74,16 +78,30 @@ def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> Li
         return []
     import pyarrow.parquet as pq
 
-    common = None
+    def _family_common(fam_dir: Path) -> set:
-    for p in sorted(feat_dir.glob("symbol=*.parquet")):
+        common = None
-        try:
+        for p in sorted(fam_dir.glob("symbol=*.parquet")):
-            cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS)
+            try:
-        except Exception:  # pragma: no cover - skip unreadable files
+                cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS)
-            continue
+            except Exception:  # pragma: no cover - skip unreadable files
-        common = cols if common is None else (common & cols)
+                continue
-        if not common:
+            common = cols if common is None else (common & cols)
-            break
+            if not common:
-    return sorted(common) if common else []
+                break
+        return common or set()
+
+    common: set = set()
+    # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet
+    for fam in ("ta", "sp"):
+        fam_dir = feat_dir / f"family={fam}"
+        if fam_dir.is_dir():
+            common |= _family_common(fam_dir)
+    # legacy flat: features/market=*/timeframe=*/symbol=*.parquet
+    if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists():
+        pass  # family layout already covered
+    else:
+        common |= _family_common(feat_dir)
+    return sorted(common)
 
 
 class DropAllNaN(processor_module.Processor):

code/tac-qlib/tac_qlib/data/config.py

@@ -6,10 +6,11 @@ The lake is a hive-partitioned parquet store (see ``tac-engine/skills/tradeac-la
     ├── market=US/
     │   └── timeframe=1d/
     │       └── symbol=AAPL.parquet        # OHLCV bars: t, date, o, h, l, c, v, n, vw
-    ├── features/                          # ta-lib indicators, wide format
+    ├── features/                          # indicators, wide format, family tier
     │   └── market=US/
     │       └── timeframe=1d/
-    │           └── symbol=AAPL.parquet    # t, sma_5, sma_20, rsi_14, ...
+    │           ├── family=ta/symbol=AAPL.parquet  # t, sma_5, sma_20, rsi_14, ...
+    │           └── family=sp/symbol=AAPL.parquet  # t, sp_ou_*, sp_hmm_*, ...
     ├── calendar.parquet                   # trading days per market
     ├── coverage.parquet                   # per (market,timeframe,symbol) loaded windows
     └── symbols.parquet                    # asset master
@@ -107,8 +108,34 @@ class LakeConfig:
         return self.lake_root / "features" / f"market={self.market}" / f"timeframe={timeframe}"
 
     def features_path(self, timeframe: str, symbol: str) -> Path:
+        # Legacy flat path (no family tier). Prefer `load_features` which
+        # resolves the family=ta|sp partition layout.
         return self.features_dir(timeframe) / f"symbol={str(symbol).upper()}.parquet"
 
+    def load_features(self, timeframe: str, symbol: str) -> pd.DataFrame:
+        """All feature columns for a symbol, merging the `family=ta` and
+        `family=sp` partitions by timestamp. Returns an empty frame when no
+        feature files exist (legacy flat layout falls back transparently)."""
+        sym = str(symbol).upper()
+        frames = []
+        for family in ("ta", "sp"):
+            p = self.features_dir(timeframe) / f"family={family}" / f"symbol={sym}.parquet"
+            if p.exists():
+                frames.append(pd.read_parquet(p))
+        if not frames:
+            flat = self.features_dir(timeframe) / f"symbol={sym}.parquet"
+            if flat.exists():
+                return pd.read_parquet(flat)
+            return pd.DataFrame()
+        if len(frames) == 1:
+            return frames[0]
+        merged = frames[0]
+        for extra in frames[1:]:
+            merged = merged.merge(extra, on="t", how="outer", suffixes=("", "_dup"))
+            for c in [c for c in merged.columns if c.endswith("_dup")]:
+                merged = merged.drop(columns=c)
+        return merged
+
     def calendar_path(self) -> Path:
         return self.lake_root / "calendar.parquet"
 

code/tac-qlib/tac_qlib/data/providers.py

@@ -173,8 +173,7 @@ class LakeFeatureProvider(FeatureProvider):
     def _load_feature_df(self, instrument: str, timeframe: str) -> pd.DataFrame:
         key = (instrument, timeframe)
         if key not in self._feature_cache:
-            p = self.cfg.features_path(timeframe, instrument)
+            self._feature_cache[key] = self.cfg.load_features(timeframe, instrument)
-            self._feature_cache[key] = pd.read_parquet(p) if p.exists() else pd.DataFrame()
         return self._feature_cache[key]
 
     @staticmethod