start experiment 52 (exp/52-walk-forward-re-validation-of-the-3-best)

code/tac-qlib/tac_qlib/contrib/model/__init__.py

@@ -0,0 +1,4 @@
+from .rank_ensemble import RankICEnsembleLGBModel  # noqa: F401
+from .rank_gbdt import RankICLGBModel, rankic_feval  # noqa: F401
+
+__all__ = ["RankICLGBModel", "rankic_feval", "RankICEnsembleLGBModel"]

code/tac-qlib/tac_qlib/contrib/model/rank_ensemble.py

@@ -0,0 +1,189 @@
+"""Seed-ensembled LightGBM that early-stops on cross-sectional RankIC.
+
+``RankICEnsembleLGBModel`` wraps ``RankICLGBModel`` (per-day RankIC feval +
+``metric='None'`` + ``first_metric_only`` early stopping) over a seed ensemble:
+one sub-model is trained per seed with identical hyper-parameters, and
+predictions are averaged across seeds. This is the model class the
+``tac-rd-rank-ensemble-isolated`` reference run wires into its workflow
+(``module_path: tac_qlib.contrib.model.rank_ensemble``).
+
+The ensemble inherits the RankIC early-stopping behaviour of the single-seed
+model (valid RankIC drives the stopping iteration) while the seed averaging
+stabilizes the prediction against any single seed's early-stopping path.
+
+Training is parallelized: the seed sub-models train in a thread pool —
+``lgb.train`` is C++ and releases the GIL, so concurrent seeds do not block on
+the GIL (5 seeds ~40min/5 on this box). Measured on a 6-physical-core / 12 SMT
+host: the seeds scale ~2x, not linearly — the runs are memory-bandwidth bound
+and each Booster caps its threads at ``cores // workers`` so 5 concurrent
+boosters don't oversubscribe; larger-core hosts scale better. The qlib data
+pipeline is warmed once on the calling thread (fills the handler cache), and
+each worker then prepares its **own** ``lgb.Dataset`` (independent handle, so
+no concurrent ``construct()`` on a shared handle — LightGBM's ``Dataset`` is
+not thread-safe to build). qlib's ``R`` recorder is also not thread-safe, so
+the per-seed evaluation curves are logged on the calling thread after the pool
+finishes.
+
+Wired into a workflow yaml like:
+
+    model:
+        class: RankICEnsembleLGBModel
+        module_path: tac_qlib.contrib.model.rank_ensemble
+        kwargs:
+            loss: mse
+            learning_rate: 0.02
+            num_leaves: 31
+            n_estimators: 3000
+            num_boost_round: 3000
+            early_stopping_rounds: 200
+            min_data_in_leaf: 20
+            lambda_l2: 0.5
+            colsample_bytree: 0.8
+            subsample: 0.8
+            subsample_freq: 1
+            reg_alpha: 0.1
+            reg_lambda: 1.0
+            seeds: "42,7,2026,99,123"
+            parallel: 5
+
+Any ``**kwargs`` other than ``seeds``/``parallel`` are forwarded unchanged to
+every ``RankICLGBModel`` sub-model (same params, different ``seed``).
+"""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import List, Optional
+
+import pandas as pd
+
+from qlib.data.dataset import DatasetH
+from qlib.data.dataset.handler import DataHandlerLP
+
+from tac_qlib.contrib.model.rank_gbdt import RankICLGBModel
+
+__all__ = ["RankICEnsembleLGBModel"]
+
+
+class RankICEnsembleLGBModel(RankICLGBModel):
+    """Seed ensemble of RankIC-early-stopping LightGBM models.
+
+    Parameters
+    ----------
+    seeds : comma-separated integers, one sub-model per seed.
+    parallel : number of seeds to train concurrently. ``0`` (default) = auto
+        (all seeds, bounded by the available cores); ``1`` = sequential.
+    **kwargs : forwarded to every ``RankICLGBModel`` sub-model (model
+        hyper-parameters). ``seeds``/``parallel`` are consumed here and not
+        forwarded.
+    """
+
+    def __init__(self, seeds: str = "42", parallel: int = 0, **kwargs):
+        self.seeds = [int(s.strip()) for s in str(seeds).split(",") if s.strip()]
+        if not self.seeds:
+            raise ValueError("seeds must contain at least one integer")
+        self.parallel = int(parallel)
+        # drop seed/parallel handling from the base kwargs, keep everything else
+        self._model_kwargs = dict(kwargs)
+        super().__init__(**self._model_kwargs)
+        self._models: List[RankICLGBModel] = []
+
+    # --------------------------------------------------------------- helpers
+    @staticmethod
+    def _cores() -> int:
+        try:
+            return max(1, len(os.sched_getaffinity(0)))
+        except AttributeError:
+            return max(1, os.cpu_count() or 1)
+
+    def _worker_count(self) -> int:
+        if self.parallel > 0:
+            return min(len(self.seeds), self.parallel)
+        return min(len(self.seeds), self._cores())
+
+    # ------------------------------------------------------------------ fit
+    def fit(
+        self,
+        dataset: DatasetH,
+        num_boost_round: Optional[int] = None,
+        early_stopping_rounds: Optional[int] = None,
+        verbose_eval: int = 20,
+        evals_result=None,
+        reweighter=None,
+        **kwargs,
+    ):
+        """Train one RankICLGBModel per seed and keep them for prediction.
+
+        The qlib data pipeline is warmed once on this thread (handler cache),
+        then each seed sub-model trains in a parallel worker thread on its own
+        ``lgb.Dataset`` (LightGBM releases the GIL in ``lgb.train``). Evals
+        are logged on this thread after the pool (qlib's ``R`` is not
+        thread-safe).
+        """
+        n_round = num_boost_round or self.num_boost_round
+        n_es = early_stopping_rounds or self.early_stopping_rounds
+
+        if len(self.seeds) == 1:
+            m = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs)
+            m.fit(
+                dataset,
+                num_boost_round=n_round,
+                early_stopping_rounds=n_es,
+                verbose_eval=verbose_eval,
+                evals_result=evals_result,
+                reweighter=reweighter,
+                **kwargs,
+            )
+            self._models = [m]
+            return
+
+        # Warm the qlib handler cache once on this thread so the workers'
+        # concurrent prepare() calls only hit cached frames (no first-write race).
+        proto = RankICLGBModel(seed=self.seeds[0], **self._model_kwargs)
+        proto._prepare_data(dataset, reweighter)
+
+        workers = self._worker_count()
+        # Cap per-Booster threads so concurrent seeds don't oversubscribe
+        # (LightGBM's num_threads=0 uses ALL cores per Booster).
+        per_booster = max(1, self._cores() // workers)
+
+        def fit_seed(seed):
+            m = RankICLGBModel(seed=seed, **self._model_kwargs)
+            if workers > 1 and "num_threads" not in m.params:
+                m.params["num_threads"] = per_booster
+            ds_l = m._prepare_data(dataset, reweighter)
+            booster, evals, names = m._train_from_datasets(
+                ds_l,
+                num_boost_round=n_round,
+                early_stopping_rounds=n_es,
+                verbose_eval=verbose_eval,
+                **kwargs,
+            )
+            m.model = booster
+            return m, evals, names
+
+        with ThreadPoolExecutor(max_workers=workers) as ex:
+            results = list(ex.map(fit_seed, self.seeds))
+
+        self._models = [m for m, _, _ in results]
+
+        # Merge + log evals on the main thread (qlib's R is not thread-safe).
+        if evals_result is not None:
+            for m, evals, names in results:
+                for k in names:
+                    for key, val in evals.get(k, {}).items():
+                        evals_result.setdefault(f"{k}.seed{m.params['seed']}", {})[key] = val
+        for m, evals, names in results:
+            self._log_evals(evals, names, prefix=f"seed{m.params['seed']}.")
+
+    # -------------------------------------------------------------- predict
+    def predict(self, dataset: DatasetH, segment="test") -> pd.Series:
+        """Average the per-seed predictions over the given segment."""
+        if not self._models:
+            raise ValueError("model is not fitted yet!")
+        preds = [m.predict(dataset, segment=segment) for m in self._models]
+        if len(preds) == 1:
+            return preds[0]
+        frame = pd.concat(preds, axis=1)
+        return frame.mean(axis=1)

code/tac-qlib/tac_qlib/contrib/model/rank_gbdt.py

@@ -0,0 +1,238 @@
+"""LGBModel variant that early-stops on cross-sectional RankIC instead of l2.
+
+Standard qlib ``LGBModel`` early-stops on the regression loss (mse). For
+cross-sectional alpha signals the quantity we actually care about is the per-day
+rank correlation (Rank IC), which mse early-stopping does not optimize for.
+Experiments on the 50-ETF lake (SP-5d 55-feature panel) show that early-stopping
+on a custom RankIC feval lifts RankIC 0.047 -> 0.075 vs. the mse-stopped model.
+
+This class reuses ``LGBModel``'s data preparation but:
+
+  - tags each ``lgb.Dataset`` with per-day query ``group`` sizes so a ranking
+    metric can be computed per trading day;
+  - injects a custom ``feval`` (mean per-day Spearman of pred vs label) into
+    ``lgb.train``; early stopping then selects the iteration that maximizes
+    RankIC on the valid set;
+  - forces ``metric='None'`` + ``first_metric_only=True`` so early-stopping
+    tracks RankIC only (not the regression loss).
+
+Wired into a workflow yaml like:
+
+    model:
+        class: RankICLGBModel
+        module_path: tac_qlib.contrib.model.rank_gbdt
+        kwargs:
+            loss: mse
+            learning_rate: 0.03
+            num_leaves: 31
+            n_estimators: 500
+            ...
+
+The rank feval is used for early-stopping selection only; the objective stays
+the configured loss (default mse). Set ``rank_eval=False`` to fall back to the
+plain LGBModel behaviour (early-stop on the loss).
+
+Generic: works for any cross-sectional panel whose qlib dataset index has a
+``datetime`` level (each level value = one query group). The per-day groups are
+derived automatically, so no universe-specific configuration is needed.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+import lightgbm as lgb
+
+from qlib.data.dataset import DatasetH
+from qlib.data.dataset.handler import DataHandlerLP
+from qlib.contrib.model.gbdt import LGBModel
+from qlib.workflow import R
+
+__all__ = ["RankICLGBModel", "rankic_feval"]
+
+
+def _group_averaged_rank(values: np.ndarray, gid: np.ndarray, offs: np.ndarray) -> np.ndarray:
+    """Averaged (tie-corrected) rank of ``values`` within each group, vectorized.
+
+    ``gid`` maps each row to its group id; ``offs`` holds the cumulative row
+    offsets so that group ``i`` occupies rows ``[offs[i], offs[i+1])``. Returns
+    the same result as ``pandas.Series.rank(method='average')`` applied per
+    group, but in one pass (``np.lexsort`` is the only non-linear step).
+    """
+    n = len(values)
+    order = np.lexsort((values, gid))
+    ord_rank = np.empty(n, dtype=np.float64)
+    ord_rank[order] = np.arange(n, dtype=np.float64) - offs[gid[order]] + 1.0
+    sg = gid[order]
+    sv = values[order]
+    newblock = np.empty(n, dtype=bool)
+    newblock[0] = True
+    newblock[1:] = (sg[1:] != sg[:-1]) | (sv[1:] != sv[:-1])
+    blockid = np.cumsum(newblock) - 1
+    block_mean = np.bincount(blockid, weights=ord_rank[order]) / np.bincount(blockid)
+    out = np.empty(n)
+    out[order] = block_mean[blockid]
+    return out
+
+
+def _per_day_spearman(preds: np.ndarray, labels: np.ndarray, group: np.ndarray) -> float:
+    """Mean per-day Spearman rank correlation of preds vs labels.
+
+    ``group`` holds the number of rows of each trading day (query group), in
+    order. Days with <3 valid rows or a constant pred/label are skipped.
+
+    Vectorized: per-day Spearman == Pearson of the per-day rank transforms,
+    and the Pearson moments (``sum``, ``sum`` of products/squares) aggregate
+    over each day with ``np.bincount``. Runs ~10x faster than the per-day
+    ``pd.Series.rank()`` loop that preceded it — this feval is invoked on the
+    train and valid panels every boosting round, per seed.
+    """
+    if group is None or len(group) == 0:
+        return 0.0
+    offs = np.concatenate([[0], np.cumsum(group.astype(int))])
+    gid = np.repeat(np.arange(len(group)), group.astype(int))
+    rp = _group_averaged_rank(preds, gid, offs)
+    rl = _group_averaged_rank(labels, gid, offs)
+    n_g = group.astype(float)
+    s_p = np.bincount(gid, weights=rp)
+    s_l = np.bincount(gid, weights=rl)
+    s_pl = np.bincount(gid, weights=rp * rl)
+    s_pp = np.bincount(gid, weights=rp * rp)
+    s_ll = np.bincount(gid, weights=rl * rl)
+    cov = n_g * s_pl - s_p * s_l
+    var_p = n_g * s_pp - s_p ** 2
+    var_l = n_g * s_ll - s_l ** 2
+    denom = np.sqrt(var_p * var_l)
+    valid = (n_g >= 3) & (denom > 0)
+    corr = np.where(valid, cov / np.where(denom == 0, 1, denom), 0.0)
+    return float(corr[valid].mean()) if valid.any() else 0.0
+
+
+def rankic_feval(preds, dataset):
+    """LightGBM feval: mean RankIC (higher is better in lgb convention)."""
+    labels = dataset.get_label()
+    group = dataset.get_group()
+    ric = _per_day_spearman(preds, labels, group)
+    return "rankic", ric, True  # (name, value, higher_is_better)
+
+
+class RankICLGBModel(LGBModel):
+    """LGBModel that early-stops on per-day RankIC via a custom feval."""
+
+    def __init__(self, rank_eval: bool = True, **kwargs):
+        super().__init__(**kwargs)
+        self.rank_eval = rank_eval
+
+    def _prepare_data(self, dataset: DatasetH, reweighter=None) -> List[Tuple[lgb.Dataset, str]]:
+        ds_l = []
+        assert "train" in dataset.segments
+        for key in ["train", "valid"]:
+            if key in dataset.segments:
+                df = dataset.prepare(key, col_set=["feature", "label"], data_key=DataHandlerLP.DK_L)
+                if df.empty:
+                    raise ValueError("Empty data from dataset, please check your dataset config.")
+                x, y = df["feature"], df["label"]
+                if y.values.ndim == 2 and y.values.shape[1] == 1:
+                    y = np.squeeze(y.values)
+                else:
+                    raise ValueError("LightGBM doesn't support multi-label training")
+
+                if reweighter is None:
+                    w = None
+                elif hasattr(reweighter, "reweight"):
+                    w = reweighter.reweight(df)
+                else:
+                    raise ValueError("Unsupported reweighter type.")
+
+                # per-day query groups: each trading day is one group
+                if self.rank_eval and isinstance(df.index, pd.MultiIndex) and "datetime" in df.index.names:
+                    group = df.groupby(level="datetime").size().to_numpy(dtype=np.int32)
+                else:
+                    group = None
+
+                d = lgb.Dataset(x.values, label=y, weight=w, group=group, free_raw_data=False)
+                ds_l.append((d, key))
+        return ds_l
+
+    def _train_from_datasets(
+        self,
+        ds_l: List[Tuple[lgb.Dataset, str]],
+        num_boost_round: Optional[int] = None,
+        early_stopping_rounds: Optional[int] = None,
+        verbose_eval: int = 20,
+        evals_result=None,
+        **kwargs,
+    ) -> Tuple[lgb.Booster, dict, List[str]]:
+        """Train a Booster from already-prepared ``lgb.Dataset`` objects.
+
+        Pure training — no ``R.log_metrics`` — so it can be called from worker
+        threads (qlib's ``R`` recorder is not thread-safe; the caller decides
+        when/where to log). Returns ``(booster, evals_result, segment_names)``.
+        """
+        if evals_result is None:
+            evals_result = {}
+        ds, names = list(zip(*ds_l))
+
+        callbacks = [
+            lgb.early_stopping(
+                self.early_stopping_rounds if early_stopping_rounds is None else early_stopping_rounds
+            ),
+            lgb.log_evaluation(period=verbose_eval),
+            lgb.record_evaluation(evals_result),
+        ]
+        if self.rank_eval:
+            # early-stopping must be driven ONLY by the RankIC feval, not l2.
+            # metric='None' suppresses the default l2 metric; first_metric_only
+            # makes early_stopping track the single remaining (rankic) metric.
+            self.params["metric"] = "None"
+            self.params["first_metric_only"] = True
+            feval = rankic_feval
+        else:
+            self.params.pop("metric", None)
+            self.params.pop("first_metric_only", None)
+            feval = None
+
+        booster = lgb.train(
+            self.params,
+            ds[0],
+            num_boost_round=self.num_boost_round if num_boost_round is None else num_boost_round,
+            valid_sets=ds,
+            valid_names=names,
+            feval=feval,
+            callbacks=callbacks,
+            **kwargs,
+        )
+        return booster, evals_result, list(names)
+
+    def _log_evals(self, evals_result, names: List[str], prefix: str = "") -> None:
+        """Log recorded evaluation curves to qlib's active recorder."""
+        for k in names:
+            for key, val in evals_result.get(k, {}).items():
+                name = f"{prefix}{key}.{k}"
+                for epoch, m in enumerate(val):
+                    R.log_metrics(**{name.replace("@", "_"): m}, step=epoch)
+
+    def fit(
+        self,
+        dataset: DatasetH,
+        num_boost_round: Optional[int] = None,
+        early_stopping_rounds: Optional[int] = None,
+        verbose_eval: int = 20,
+        evals_result=None,
+        reweighter=None,
+        **kwargs,
+    ):
+        if evals_result is None:
+            evals_result = {}
+        ds_l = self._prepare_data(dataset, reweighter)
+        self.model, evals_result, names = self._train_from_datasets(
+            ds_l,
+            num_boost_round=num_boost_round,
+            early_stopping_rounds=early_stopping_rounds,
+            verbose_eval=verbose_eval,
+            evals_result=evals_result,
+            **kwargs,
+        )
+        self._log_evals(evals_result, names)