start experiment 42 (exp/42-q10-hmm-regime-overlay-entry-gate-on-sph)

code/tac-qlib/tac_qlib/contrib/data/handler.py

@@ -64,9 +64,13 @@ def check_transform_proc(proc_l, fit_start_time, fit_end_time):
 
 
 def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> List[str]:
-    """Discover ta-lib columns present in *every* features parquet file of the lake.
+    """Discover feature columns present in *every* feature file of the lake.
 
-    Returns sorted field names (without the ``$`` prefix). Empty if no features are persisted.
+    Walks the `family=ta|sp` partition layout (plus any legacy flat files).
+    TA and SP columns are disjoint by construction, so the common set is
+    computed per family (columns shared by all symbol files of that family),
+    then the per-family results are unioned. Returns sorted field names
+    (without the ``$`` prefix). Empty if no features are persisted.
     """
     cfg = LakeConfig(lake_root, market)
     feat_dir = cfg.features_dir(timeframe)
@@ -74,16 +78,30 @@ def get_common_feature_fields(lake_root=None, market="US", timeframe="1d") -> Li
         return []
     import pyarrow.parquet as pq
 
-    common = None
-    for p in sorted(feat_dir.glob("symbol=*.parquet")):
-        try:
-            cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS)
-        except Exception:  # pragma: no cover - skip unreadable files
-            continue
-        common = cols if common is None else (common & cols)
-        if not common:
-            break
-    return sorted(common) if common else []
+    def _family_common(fam_dir: Path) -> set:
+        common = None
+        for p in sorted(fam_dir.glob("symbol=*.parquet")):
+            try:
+                cols = set(pq.read_schema(p).names) - set(NON_FEATURE_COLUMNS)
+            except Exception:  # pragma: no cover - skip unreadable files
+                continue
+            common = cols if common is None else (common & cols)
+            if not common:
+                break
+        return common or set()
+
+    common: set = set()
+    # family tier: features/market=*/timeframe=*/family=*/symbol=*.parquet
+    for fam in ("ta", "sp"):
+        fam_dir = feat_dir / f"family={fam}"
+        if fam_dir.is_dir():
+            common |= _family_common(fam_dir)
+    # legacy flat: features/market=*/timeframe=*/symbol=*.parquet
+    if (feat_dir / "family=ta").exists() or (feat_dir / "family=sp").exists():
+        pass  # family layout already covered
+    else:
+        common |= _family_common(feat_dir)
+    return sorted(common)
 
 
 class DropAllNaN(processor_module.Processor):