ch11: signal-quality gate REFUTED — walk-forward workflow shows gate harmful (exp 61-67, EVIDENCE#053)
This commit is contained in:
zhaoli
@@ -0,0 +1,191 @@
|
||||
"""Diagnose exactly why the scripted test and workflow give different results.
|
||||
|
||||
Compares the same pred.pkl through:
|
||||
1. Script logic (weekly rebalance, equal-weight, hold-through-week, zero cost)
|
||||
2. Workflow logic (PortAnaRecord daily backtest, TopkDropout-like)
|
||||
|
||||
Isolates the effect of:
|
||||
A. Weekly vs daily position evaluation
|
||||
B. Equal weight vs risk_degree sizing
|
||||
C. Hold-through-week vs daily top-k re-ranking
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import json, pathlib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
LAKE_ROOT = "/home/data/lake"
|
||||
OUT = pathlib.Path("/app/experiments/book/data/diag_script_vs_wf")
|
||||
|
||||
WINDOWS = [
|
||||
{"label": "2026", "start": "2026-01-04", "end": "2026-08-19",
|
||||
"pred": f"{LAKE_ROOT}/mlruns/62/3771f96eb1b74365aeae966af7aec5a3/artifacts/pred.pkl"},
|
||||
{"label": "2025", "start": "2025-01-02", "end": "2025-12-31",
|
||||
"pred": f"{LAKE_ROOT}/mlruns/62/c57c6a8370cc48619d7cdd2bd109b76a/artifacts/pred.pkl"},
|
||||
{"label": "2024", "start": "2024-01-02", "end": "2024-12-31",
|
||||
"pred": f"{LAKE_ROOT}/mlruns/62/97cf5f282e6f4e699443e38d9bfb40fd/artifacts/pred.pkl"},
|
||||
{"label": "2023", "start": "2023-01-03", "end": "2023-12-29",
|
||||
"pred": f"{LAKE_ROOT}/mlruns/62/11b9b65ea4e14b3f8ce50d244da0412e/artifacts/pred.pkl"},
|
||||
{"label": "2021", "start": "2021-01-04", "end": "2021-12-31",
|
||||
"pred": f"{LAKE_ROOT}/mlruns/62/af3034e5910348a382f2ad1e1741f17c/artifacts/pred.pkl"},
|
||||
]
|
||||
|
||||
SYMS = [
|
||||
"SPY","QQQ","DIA","IWM","MDY","VTI","VOO","VEA","VWO","VT","EFA","EEM",
|
||||
"TLT","IEF","SHY","AGG","BND","LQD","HYG","JNK","EMB","GLD","SLV",
|
||||
"USO","UNG","DBA","DBC","XLK","XLF","XLE","XLV","XLI","XLY","XLP",
|
||||
"XLU","XLB","XLRE","ARKK","SMH","SOXX","IBB","XBI","ITA","XAR",
|
||||
"ICLN","TAN","FDN","IGV","ESPO","REM",
|
||||
]
|
||||
|
||||
|
||||
def load_pred(path):
|
||||
df = pd.read_pickle(path)
|
||||
s = df["score"] if isinstance(df, pd.DataFrame) and "score" in df.columns else df.iloc[:, 0] if isinstance(df, pd.DataFrame) else df
|
||||
idx = s.index
|
||||
new_dt = pd.to_datetime(idx.get_level_values(0)).normalize()
|
||||
s.index = pd.MultiIndex.from_arrays([new_dt, idx.get_level_values(1)], names=idx.names)
|
||||
return s
|
||||
|
||||
|
||||
def load_closes(start, end):
|
||||
from tac_qlib.data.config import LakeConfig, resolve_lake_root
|
||||
cfg = LakeConfig(resolve_lake_root(LAKE_ROOT), "US")
|
||||
closes = {}
|
||||
for sym in SYMS:
|
||||
p = cfg.bar_path("1d", sym)
|
||||
if not p.exists(): continue
|
||||
try:
|
||||
df = pd.read_parquet(p)
|
||||
except: continue
|
||||
if not len(df): continue
|
||||
tcol = df["t"] if "t" in df.columns else df["date"]
|
||||
ts = pd.to_datetime(tcol)
|
||||
df = df.assign(_t=ts).set_index("_t").sort_index()
|
||||
warmup = pd.Timestamp(start) - pd.Timedelta(days=60)
|
||||
df = df.loc[warmup:end]
|
||||
if len(df) >= 22:
|
||||
closes[sym] = df["c"]
|
||||
return pd.DataFrame(closes)
|
||||
|
||||
|
||||
def strategy_script(pred, closes, start, end, topk=10, risk_degree=1.0):
|
||||
"""Mimics the scripted test: weekly rebalance, hold all week."""
|
||||
ret_df = closes.pct_change()
|
||||
ret_df.index = pd.to_datetime(ret_df.index).normalize()
|
||||
dt_idx = pred.index.get_level_values(0)
|
||||
trade_dates = sorted(dt_idx[(dt_idx >= start) & (dt_idx <= end)].unique())
|
||||
equity = 1_000_000.0
|
||||
holdings = []
|
||||
prev_week = None
|
||||
daily_eq = []
|
||||
for d in trade_dates:
|
||||
try:
|
||||
day_scores = pred.loc[d]
|
||||
except KeyError:
|
||||
daily_eq.append(equity)
|
||||
prev_scores = None
|
||||
continue
|
||||
if isinstance(day_scores, pd.DataFrame):
|
||||
day_scores = day_scores.iloc[:, 0]
|
||||
day_scores = day_scores.dropna().sort_values(ascending=False)
|
||||
cur_week = (d.isocalendar()[0], d.isocalendar()[1])
|
||||
if cur_week != prev_week or not holdings:
|
||||
holdings = list(day_scores.index[:topk])
|
||||
ret_row = ret_df.loc[d] if d in ret_df.index else None
|
||||
if ret_row is not None and holdings:
|
||||
wts = np.array([risk_degree / len(holdings)] * len(holdings))
|
||||
rets = ret_row.reindex(holdings).fillna(0).values
|
||||
equity *= (1 + (wts * rets).sum())
|
||||
daily_eq.append(equity)
|
||||
prev_week = cur_week
|
||||
return pd.Series(daily_eq, index=trade_dates)
|
||||
|
||||
|
||||
def strategy_daily_topk(pred, closes, start, end, topk=10, risk_degree=1.0):
|
||||
"""Mimics PortAnaRecord: re-rank every day, hold top-k."""
|
||||
ret_df = closes.pct_change()
|
||||
ret_df.index = pd.to_datetime(ret_df.index).normalize()
|
||||
dt_idx = pred.index.get_level_values(0)
|
||||
trade_dates = sorted(dt_idx[(dt_idx >= start) & (dt_idx <= end)].unique())
|
||||
equity = 1_000_000.0
|
||||
daily_eq = []
|
||||
for d in trade_dates:
|
||||
try:
|
||||
day_scores = pred.loc[d]
|
||||
except KeyError:
|
||||
daily_eq.append(equity)
|
||||
continue
|
||||
if isinstance(day_scores, pd.DataFrame):
|
||||
day_scores = day_scores.iloc[:, 0]
|
||||
day_scores = day_scores.dropna().sort_values(ascending=False)
|
||||
holdings = list(day_scores.index[:topk])
|
||||
ret_row = ret_df.loc[d] if d in ret_df.index else None
|
||||
if ret_row is not None and holdings:
|
||||
wts = np.array([risk_degree / len(holdings)] * len(holdings))
|
||||
rets = ret_row.reindex(holdings).fillna(0).values
|
||||
equity *= (1 + (wts * rets).sum())
|
||||
daily_eq.append(equity)
|
||||
return pd.Series(daily_eq, index=trade_dates)
|
||||
|
||||
|
||||
def metrics(eq):
|
||||
if len(eq) < 2:
|
||||
return {"ann_ret": 0, "sharpe": 0, "maxDD": 0}
|
||||
rets = eq.pct_change().dropna()
|
||||
ann_ret = float((eq.iloc[-1] / eq.iloc[0]) ** (252 / max(len(eq), 1)) - 1)
|
||||
vol = float(rets.std() * (252 ** 0.5)) if len(rets) > 1 else 0
|
||||
sharpe = ann_ret / vol if vol > 0 else 0
|
||||
peak = eq.cummax()
|
||||
dd = (eq - peak) / peak
|
||||
return {"ann_ret": round(ann_ret, 4), "sharpe": round(sharpe, 4), "maxDD": round(float(dd.min()), 4)}
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
results = []
|
||||
for w in WINDOWS:
|
||||
print(f"\n=== {w['label']} ({w['start']} to {w['end']}) ===")
|
||||
pred = load_pred(w["pred"])
|
||||
closes = load_closes(w["start"], w["end"])
|
||||
print(f" pred dates: {pred.index.get_level_values(0).min()} to {pred.index.get_level_values(0).max()}")
|
||||
print(f" close dates: {closes.index.min()} to {closes.index.max()}")
|
||||
print(f" symbols in close: {closes.shape[1]}")
|
||||
|
||||
# Script: weekly, equal weight (risk_degree=1.0)
|
||||
eq_weekly_100 = strategy_script(pred, closes, w["start"], w["end"], topk=10, risk_degree=1.0)
|
||||
m_weekly_100 = metrics(eq_weekly_100)
|
||||
|
||||
# Script: weekly, 95% risk degree
|
||||
eq_weekly_95 = strategy_script(pred, closes, w["start"], w["end"], topk=10, risk_degree=0.95)
|
||||
m_weekly_95 = metrics(eq_weekly_95)
|
||||
|
||||
# Daily top-k: re-rank daily, equal weight
|
||||
eq_daily_100 = strategy_daily_topk(pred, closes, w["start"], w["end"], topk=10, risk_degree=1.0)
|
||||
m_daily_100 = metrics(eq_daily_100)
|
||||
|
||||
# Daily top-k: re-rank daily, 95%
|
||||
eq_daily_95 = strategy_daily_topk(pred, closes, w["start"], w["end"], topk=10, risk_degree=0.95)
|
||||
m_daily_95 = metrics(eq_daily_95)
|
||||
|
||||
row = {
|
||||
"year": w["label"],
|
||||
"script_weekly_100": m_weekly_100,
|
||||
"script_weekly_95": m_weekly_95,
|
||||
"daily_topk_100": m_daily_100,
|
||||
"daily_topk_95": m_daily_95,
|
||||
}
|
||||
results.append(row)
|
||||
print(f" Script weekly 100%: ann={m_weekly_100['ann_ret']:+.1%} sharpe={m_weekly_100['sharpe']:.2f} maxDD={m_weekly_100['maxDD']:.1%}")
|
||||
print(f" Script weekly 95%: ann={m_weekly_95['ann_ret']:+.1%} sharpe={m_weekly_95['sharpe']:.2f} maxDD={m_weekly_95['maxDD']:.1%}")
|
||||
print(f" Daily topk 100%: ann={m_daily_100['ann_ret']:+.1%} sharpe={m_daily_100['sharpe']:.2f} maxDD={m_daily_100['maxDD']:.1%}")
|
||||
print(f" Daily topk 95%: ann={m_daily_95['ann_ret']:+.1%} sharpe={m_daily_95['sharpe']:.2f} maxDD={m_daily_95['maxDD']:.1%}")
|
||||
|
||||
with open(OUT / "diagnosis.json", "w") as f:
|
||||
json.dump(results, f, indent=2, default=str)
|
||||
print(f"\nSaved to {OUT / 'diagnosis.json'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user