"""Diagnose the script-vs-workflow gap properly. Three strategies compared: A. Script logic: weekly rebalance, equal-weight, hold through week B. Weekly rebalance (qlib engine behavior): same as script but with risk_degree C. Daily re-rank: re-select top-k every day (wrong model) Root cause was (C) — we were modeling daily re-ranking which neither the script nor the qlib engine actually does. """ from __future__ import annotations import json, pathlib import numpy as np import pandas as pd LAKE_ROOT = "/home/data/lake" OUT = pathlib.Path("/app/experiments/book/data/diag_script_vs_wf") WINDOWS = [ {"label": "2026", "start": "2026-01-04", "end": "2026-08-19", "pred": f"{LAKE_ROOT}/mlruns/62/3771f96eb1b74365aeae966af7aec5a3/artifacts/pred.pkl"}, {"label": "2025", "start": "2025-01-02", "end": "2025-12-31", "pred": f"{LAKE_ROOT}/mlruns/62/c57c6a8370cc48619d7cdd2bd109b76a/artifacts/pred.pkl"}, {"label": "2024", "start": "2024-01-02", "end": "2024-12-31", "pred": f"{LAKE_ROOT}/mlruns/62/97cf5f282e6f4e699443e38d9bfb40fd/artifacts/pred.pkl"}, {"label": "2023", "start": "2023-01-03", "end": "2023-12-29", "pred": f"{LAKE_ROOT}/mlruns/62/11b9b65ea4e14b3f8ce50d244da0412e/artifacts/pred.pkl"}, {"label": "2021", "start": "2021-01-04", "end": "2021-12-31", "pred": f"{LAKE_ROOT}/mlruns/62/af3034e5910348a382f2ad1e1741f17c/artifacts/pred.pkl"}, ] SYMS = [ "SPY","QQQ","DIA","IWM","MDY","VTI","VOO","VEA","VWO","VT","EFA","EEM", "TLT","IEF","SHY","AGG","BND","LQD","HYG","JNK","EMB","GLD","SLV", "USO","UNG","DBA","DBC","XLK","XLF","XLE","XLV","XLI","XLY","XLP", "XLU","XLB","XLRE","ARKK","SMH","SOXX","IBB","XBI","ITA","XAR", "ICLN","TAN","FDN","IGV","ESPO","REM", ] def load_pred(path): df = pd.read_pickle(path) s = df["score"] if isinstance(df, pd.DataFrame) and "score" in df.columns else df.iloc[:, 0] if isinstance(df, pd.DataFrame) else df idx = s.index new_dt = pd.to_datetime(idx.get_level_values(0)).normalize() s.index = pd.MultiIndex.from_arrays([new_dt, idx.get_level_values(1)], names=idx.names) return s def load_closes(start, end): from tac_qlib.data.config import LakeConfig, resolve_lake_root cfg = LakeConfig(resolve_lake_root(LAKE_ROOT), "US") closes = {} for sym in SYMS: p = cfg.bar_path("1d", sym) if not p.exists(): continue try: df = pd.read_parquet(p) except: continue if not len(df): continue tcol = df["t"] if "t" in df.columns else df["date"] ts = pd.to_datetime(tcol) df = df.assign(_t=ts).set_index("_t").sort_index() warmup = pd.Timestamp(start) - pd.Timedelta(days=60) df = df.loc[warmup:end] if len(df) >= 22: closes[sym] = df["c"] return pd.DataFrame(closes) def strategy_weekly(pred, closes, start, end, topk=10, risk_degree=1.0, cost_bps=0): """Weekly rebalance: re-rank on first day of each ISO week, hold rest of week.""" ret_df = closes.pct_change(fill_method=None) ret_df.index = pd.to_datetime(ret_df.index).normalize() dt_idx = pred.index.get_level_values(0) trade_dates = sorted(dt_idx[(dt_idx >= start) & (dt_idx <= end)].unique()) equity = 1_000_000.0 holdings = [] prev_week = None daily_eq = [] for d in trade_dates: try: day_scores = pred.loc[d] except KeyError: daily_eq.append(equity) continue if isinstance(day_scores, pd.DataFrame): day_scores = day_scores.iloc[:, 0] day_scores = day_scores.dropna().sort_values(ascending=False) cur_week = (d.isocalendar()[0], d.isocalendar()[1]) if cur_week != prev_week: # Rebalance: compute cost of turnover new_holdings = list(day_scores.index[:topk]) if holdings and cost_bps > 0: sold = set(holdings) - set(new_holdings) bought = set(new_holdings) - set(holdings) turnover = (len(sold) + len(bought)) / (2 * max(len(holdings), 1)) equity *= (1 - turnover * cost_bps / 10000) holdings = new_holdings ret_row = ret_df.loc[d] if d in ret_df.index else None if ret_row is not None and holdings: wts = np.array([risk_degree / len(holdings)] * len(holdings)) rets = ret_row.reindex(holdings).fillna(0).values equity *= (1 + (wts * rets).sum()) daily_eq.append(equity) prev_week = cur_week return pd.Series(daily_eq, index=trade_dates) def strategy_daily(pred, closes, start, end, topk=10, risk_degree=1.0, cost_bps=0): """Daily re-rank: re-select top-k every day (wrong model — what we incorrectly tested).""" ret_df = closes.pct_change(fill_method=None) ret_df.index = pd.to_datetime(ret_df.index).normalize() dt_idx = pred.index.get_level_values(0) trade_dates = sorted(dt_idx[(dt_idx >= start) & (dt_idx <= end)].unique()) equity = 1_000_000.0 holdings = [] daily_eq = [] for d in trade_dates: try: day_scores = pred.loc[d] except KeyError: daily_eq.append(equity) continue if isinstance(day_scores, pd.DataFrame): day_scores = day_scores.iloc[:, 0] day_scores = day_scores.dropna().sort_values(ascending=False) new_holdings = list(day_scores.index[:topk]) if holdings and cost_bps > 0: sold = set(holdings) - set(new_holdings) bought = set(new_holdings) - set(holdings) turnover = (len(sold) + len(bought)) / (2 * max(len(holdings), 1)) equity *= (1 - turnover * cost_bps / 10000) holdings = new_holdings ret_row = ret_df.loc[d] if d in ret_df.index else None if ret_row is not None and holdings: wts = np.array([risk_degree / len(holdings)] * len(holdings)) rets = ret_row.reindex(holdings).fillna(0).values equity *= (1 + (wts * rets).sum()) daily_eq.append(equity) return pd.Series(daily_eq, index=trade_dates) def metrics(eq): if len(eq) < 2: return {"ann_ret": 0, "sharpe": 0, "maxDD": 0} rets = eq.pct_change().dropna() ann_ret = float((eq.iloc[-1] / eq.iloc[0]) ** (252 / max(len(eq), 1)) - 1) vol = float(rets.std() * (252 ** 0.5)) if len(rets) > 1 else 0 sharpe = ann_ret / vol if vol > 0 else 0 peak = eq.cummax() dd = (eq - peak) / peak return {"ann_ret": round(ann_ret, 4), "sharpe": round(sharpe, 4), "maxDD": round(float(dd.min()), 4)} def main(): OUT.mkdir(parents=True, exist_ok=True) results = [] for w in WINDOWS: print(f"\n=== {w['label']} ({w['start']} to {w['end']}) ===") pred = load_pred(w["pred"]) closes = load_closes(w["start"], w["end"]) print(f" pred: {pred.index.get_level_values(0).min().date()} to {pred.index.get_level_values(0).max().date()}, " f"{pred.index.get_level_values(1).nunique()} syms") print(f" close: {closes.index.min().date()} to {closes.index.max().date()}, {closes.shape[1]} syms") row = {"year": w["label"]} # A. Script: weekly, rd=1.0, zero cost eq = strategy_weekly(pred, closes, w["start"], w["end"], topk=10, risk_degree=1.0, cost_bps=0) m = metrics(eq); row["weekly_100_zc"] = m print(f" Script weekly 100% zc: ann={m['ann_ret']:+.1%} sharpe={m['sharpe']:.2f}") # B. Script: weekly, rd=0.95, zero cost eq = strategy_weekly(pred, closes, w["start"], w["end"], topk=10, risk_degree=0.95, cost_bps=0) m = metrics(eq); row["weekly_95_zc"] = m print(f" Script weekly 95% zc: ann={m['ann_ret']:+.1%} sharpe={m['sharpe']:.2f}") # C. Weekly, rd=0.95, with 5/15bp cost eq = strategy_weekly(pred, closes, w["start"], w["end"], topk=10, risk_degree=0.95, cost_bps=10) m = metrics(eq); row["weekly_95_10bp"] = m print(f" Weekly 95% 10bp cost: ann={m['ann_ret']:+.1%} sharpe={m['sharpe']:.2f}") # D. Daily re-rank, rd=1.0, zero cost (WRONG MODEL — for reference only) eq = strategy_daily(pred, closes, w["start"], w["end"], topk=10, risk_degree=1.0, cost_bps=0) m = metrics(eq); row["daily_100_zc"] = m print(f" Daily 100% zc (WRONG): ann={m['ann_ret']:+.1%} sharpe={m['sharpe']:.2f}") # E. Daily re-rank, rd=1.0, 10bp cost eq = strategy_daily(pred, closes, w["start"], w["end"], topk=10, risk_degree=1.0, cost_bps=10) m = metrics(eq); row["daily_100_10bp"] = m print(f" Daily 100% 10bp (WRONG):ann={m['ann_ret']:+.1%} sharpe={m['sharpe']:.2f}") results.append(row) with open(OUT / "diagnosis_v2.json", "w") as f: json.dump(results, f, indent=2, default=str) print(f"\nSaved to {OUT / 'diagnosis_v2.json'}") if __name__ == "__main__": main()