From 115547912d9da1720659e9018eac0ebc7a3f93f3 Mon Sep 17 00:00:00 2001 From: zhaoli Date: Thu, 20 Aug 2026 00:51:31 +0000 Subject: [PATCH] add q08 risk-limit calibration evidence (A/B results) --- evidence/q08-risklimit/RESULT.md | 35 ++ evidence/q08-risklimit/risk_calibration.json | 401 +++++++++++++++++++ queue/designs/q08_risk_limit_ab.md | 34 ++ 3 files changed, 470 insertions(+) create mode 100644 evidence/q08-risklimit/RESULT.md create mode 100644 evidence/q08-risklimit/risk_calibration.json create mode 100644 queue/designs/q08_risk_limit_ab.md diff --git a/evidence/q08-risklimit/RESULT.md b/evidence/q08-risklimit/RESULT.md new file mode 100644 index 0000000..cf1a5cf --- /dev/null +++ b/evidence/q08-risklimit/RESULT.md @@ -0,0 +1,35 @@ +# Q08 — Risk-limit A/B re-validation (trace 40) + +**Status:** DONE (verdict: REFUTED as an IR edge; safety-net value retained) + +## Input +- Reference signal: exp-26 pred, run `21afc6afdb674a399b59dd76c97628ce` (mlflow exp 25) +- Window: 2026-01-04 → 2026-08-10, Topk10 n_drop1, SPY benchmark, $1M, 5/15bp/$5 +- Tool: `rd_risk_calibrate` (A/B + sensitivity grid). Full JSON: `risk_calibration.json` + +## Candidate spec (round-3 live spec) +`{"liquidity_floor_adv": 5000000, "size_cap_pct": 0.12, "concentration_cap_pct": 0.95, "drawdown_pause_pct": 0.10}` + +## Results (net, with cost) +| Config | IR | Ann. return | Max DD | +|---|---|---|---| +| baseline (no limits) | 1.5804 | +27.50% | −6.91% | +| **candidate (5M floor + caps)** | **1.5121** | +2.20% | **−0.65%** | +| liquidity $10M | 1.5457 | +2.25% | −0.64% | + +## Findings +- **Floor binds, not a no-op**: $5M liquidity floor dropped 8 symbols — + `DBA, DBC, ESPO, FDN, REM, TAN, UNG, XAR`. +- **No IR edge from the gate**: candidate IR (1.512) is BELOW baseline (1.580). + The exp-18 direction (floor IR 0.81→0.98) does NOT reproduce on the clean-lake + reference signal. +- **Drawdown cut is pure defunding**: size_cap 0.12 × concentration 0.95 fold + the effective risk_degree to ~0.0095 → ~$9.5k deployed of $1M (~100x less). + Sensitivity grid shows both caps are no-ops (conc 20–50% identical, + size_cap 5–20% identical); only the liquidity floor moves returns, marginally. +- **Conclusion**: keep the live spec as a safety net; there is no risk-limit + gate IR edge to harvest when the signal is the bottleneck (exp-20 pattern). + +## Artifacts on this branch +- `evidence/q08-risklimit/risk_calibration.json` — full calibration dump +- `queue/designs/q08_risk_limit_ab.md` — the pre-registered design doc \ No newline at end of file diff --git a/evidence/q08-risklimit/risk_calibration.json b/evidence/q08-risklimit/risk_calibration.json new file mode 100644 index 0000000..a7bc45c --- /dev/null +++ b/evidence/q08-risklimit/risk_calibration.json @@ -0,0 +1,401 @@ +{ + "rows": [ + { + "label": "baseline (no limits)", + "mean": 0.001155, + "std": 0.011279, + "annualized_return": 0.274989, + "information_ratio": 1.580427, + "max_drawdown": -0.069145 + }, + { + "label": "liquidity $10,000,000", + "mean": 9.4e-05, + "std": 0.000942, + "annualized_return": 0.022464, + "information_ratio": 1.545736, + "max_drawdown": -0.006389 + }, + { + "label": "conc 20%", + "mean": 0.000115, + "std": 0.001168, + "annualized_return": 0.027285, + "information_ratio": 1.513718, + "max_drawdown": -0.008104 + }, + { + "label": "conc 30%", + "mean": 0.000115, + "std": 0.001168, + "annualized_return": 0.027285, + "information_ratio": 1.513718, + "max_drawdown": -0.008104 + }, + { + "label": "conc 40%", + "mean": 0.000115, + "std": 0.001168, + "annualized_return": 0.027285, + "information_ratio": 1.513718, + "max_drawdown": -0.008104 + }, + { + "label": "conc 50%", + "mean": 0.000115, + "std": 0.001168, + "annualized_return": 0.027285, + "information_ratio": 1.513718, + "max_drawdown": -0.008104 + }, + { + "label": "candidate {\"liquidity_floor_adv\": 5000000.0, \"size_cap_pct\": 0.12, \"concentration_cap_pct\": 0.95, \"drawdown_pause_pct\": 0.1}", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "size_cap 5%", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "size_cap 10%", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "size_cap 15%", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "size_cap 20%", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "liquidity $5,000,000", + "mean": 9.2e-05, + "std": 0.000943, + "annualized_return": 0.021991, + "information_ratio": 1.512051, + "max_drawdown": -0.00653 + }, + { + "label": "liquidity $1,000,000", + "mean": 9.1e-05, + "std": 0.000929, + "annualized_return": 0.021625, + "information_ratio": 1.508748, + "max_drawdown": -0.006376 + }, + { + "label": "liquidity $2,500,000", + "mean": 7.1e-05, + "std": 0.000918, + "annualized_return": 0.017, + "information_ratio": 1.199721, + "max_drawdown": -0.007158 + } + ], + "runs": { + "baseline": { + "risk": { + "mean": 0.0011554172081987572, + "std": 0.01127853762493476, + "annualized_return": 0.27498929555130425, + "information_ratio": 1.5804272791471323, + "max_drawdown": -0.06914515336341577 + }, + "applied": {} + }, + "candidate": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "size_cap 5%": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "size_cap 10%": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "size_cap 15%": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "size_cap 20%": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "conc 20%": { + "risk": { + "mean": 0.00011464156491316718, + "std": 0.0011683839517000441, + "annualized_return": 0.027284692449333788, + "information_ratio": 1.5137180903503433, + "max_drawdown": -0.008103887185240407 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "conc 30%": { + "risk": { + "mean": 0.00011464156491316718, + "std": 0.0011683839517000441, + "annualized_return": 0.027284692449333788, + "information_ratio": 1.5137180903503433, + "max_drawdown": -0.008103887185240407 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "conc 40%": { + "risk": { + "mean": 0.00011464156491316718, + "std": 0.0011683839517000441, + "annualized_return": 0.027284692449333788, + "information_ratio": 1.5137180903503433, + "max_drawdown": -0.008103887185240407 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "conc 50%": { + "risk": { + "mean": 0.00011464156491316718, + "std": 0.0011683839517000441, + "annualized_return": 0.027284692449333788, + "information_ratio": 1.5137180903503433, + "max_drawdown": -0.008103887185240407 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "liquidity $1,000,000": { + "risk": { + "mean": 9.086210454881382e-05, + "std": 0.0009290831160004576, + "annualized_return": 0.021625180882617688, + "information_ratio": 1.508747982736451, + "max_drawdown": -0.006376134679664126 + }, + "applied": { + "dropped_liquidity": [ + "ESPO" + ] + } + }, + "liquidity $2,500,000": { + "risk": { + "mean": 7.142665167642606e-05, + "std": 0.0009184775632266332, + "annualized_return": 0.016999543098989402, + "information_ratio": 1.1997208834083914, + "max_drawdown": -0.0071582979845040825 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "REM", + "XAR" + ] + } + }, + "liquidity $5,000,000": { + "risk": { + "mean": 9.239707947451976e-05, + "std": 0.0009427144352738658, + "annualized_return": 0.0219905049149357, + "information_ratio": 1.5120514373488407, + "max_drawdown": -0.006530482262119444 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "REM", + "TAN", + "UNG", + "XAR" + ] + } + }, + "liquidity $10,000,000": { + "risk": { + "mean": 9.438545151345752e-05, + "std": 0.0009420158170657147, + "annualized_return": 0.02246373746020289, + "information_ratio": 1.5457360696934006, + "max_drawdown": -0.006388809561209335 + }, + "applied": { + "dropped_liquidity": [ + "DBA", + "DBC", + "ESPO", + "FDN", + "ICLN", + "ITA", + "MDY", + "REM", + "SHY", + "TAN", + "UNG", + "XAR" + ] + } + } + }, + "candidate": { + "liquidity_floor_adv": 5000000.0, + "size_cap_pct": 0.12, + "concentration_cap_pct": 0.95, + "drawdown_pause_pct": 0.1 + } +} \ No newline at end of file diff --git a/queue/designs/q08_risk_limit_ab.md b/queue/designs/q08_risk_limit_ab.md new file mode 100644 index 0000000..2a95180 --- /dev/null +++ b/queue/designs/q08_risk_limit_ab.md @@ -0,0 +1,34 @@ +# QUEUE-08 — Risk-limit A/B re-validation: $5M liquidity floor on the exp-26 reference + +**Status:** QUEUED · **Priority:** P1 · **Effort:** tool-only (no new code) + +## Hypothesis (prove) +The $5M liquidity floor improves net IR and cuts drawdown on the **post-reset** +reference signal (pre-reset exp 18, EVIDENCE#008: net IR 0.81→0.98, cumDD +7.93%→5.44%), while size/concentration caps hurt by cutting deployed capital. +Needs re-validation on the exp-26 lineage because exp 18 is pre-clean-lake and +not comparable (EVIDENCE#009/010). Source: `book/CLAIMS.md` open question + +`book/README.md` `TODO(evidence-needed: reconciliation of exp 18 risk-limit spec +on the post-reset reference signal)`. + +## Change vs exp-26 reference (ONE variable) +- Reference: the saved exp-26 prediction (run `21afc6af…`, mlflow exp 25). +- A/B via `rd_risk_calibrate` (runs limit-vs-no-limit A/B + sensitivity grid + over size_cap_pct, concentration_cap_pct, liquidity_floor_adv) and/or + `rd_backtest` with `risk_limits` on the SAME saved `pred.pkl`: + - baseline: no limits (this must reproduce the exp-26 net +2.13% / IR 0.21); + - candidate: `{"liquidity_floor_adv": 5000000, "size_cap_pct": 0.12, + "concentration_cap_pct": 0.95, "drawdown_pause_pct": 0.10}` (round-3 spec). +- Pick the spec (B2 calibration) that keeps live ≈ backtest. + +## Acceptance +- Candidate spec: `net_IR > 0.21` AND `net_max_drawdown < 7.69%` vs no-limit on + the same pred. Size/concentration caps expected to REDUCE deployed capital + (record the direction as confirmation of exp 18). +- If the floor is a no-op (gates don't bind at this signal) → report that gates + are no-ops when the signal is the bottleneck (exp 20 pattern) as a PROVEN + clean-lake result. + +## Execution prerequisites +- None (uses saved pred + `rd_risk_calibrate`/`rd_backtest`). Trace the A/B as + an experiment; record the spec chosen for the next live round. \ No newline at end of file