From e0b31c8a2befcd4ddac8d2243fc9719fbb9e96da Mon Sep 17 00:00:00 2001 From: Sean Koval Date: Fri, 13 Feb 2026 16:52:58 -0500 Subject: [PATCH] feat(openquant): add AFML ch11-12 backtesting engine with CPCV --- crates/openquant/src/backtesting_engine.rs | 703 +++++++++++++++++++ crates/openquant/src/lib.rs | 1 + crates/openquant/tests/backtesting_engine.rs | 147 ++++ docs-site/src/data/afmlDocsState.ts | 28 + docs-site/src/data/moduleDocs.ts | 46 ++ docs-site/src/pages/api-reference.astro | 1 + 6 files changed, 926 insertions(+) create mode 100644 crates/openquant/src/backtesting_engine.rs create mode 100644 crates/openquant/tests/backtesting_engine.rs diff --git a/crates/openquant/src/backtesting_engine.rs b/crates/openquant/src/backtesting_engine.rs new file mode 100644 index 0000000..64b8497 --- /dev/null +++ b/crates/openquant/src/backtesting_engine.rs @@ -0,0 +1,703 @@ +//! Backtesting engine with walk-forward, purged CV, and CPCV support. +//! +//! AFML Chapter 11 framing is implemented through explicit diagnostics that +//! force provenance and bias-control metadata into each run, while Chapter 12 +//! split logic is represented through WF/CV/CPCV mode-specific pathways. + +use chrono::NaiveDateTime; +use std::collections::HashMap; + +#[derive(Debug, Clone, PartialEq)] +pub struct BacktestSafeguards { + pub survivorship_bias_control: String, + pub look_ahead_control: String, + pub data_mining_control: String, + pub cost_assumption: String, + pub multiple_testing_control: String, +} + +impl BacktestSafeguards { + pub fn validate(&self) -> Result<(), String> { + if self.survivorship_bias_control.trim().is_empty() { + return Err("survivorship_bias_control cannot be empty".to_string()); + } + if self.look_ahead_control.trim().is_empty() { + return Err("look_ahead_control cannot be empty".to_string()); + } + if self.data_mining_control.trim().is_empty() { + return Err("data_mining_control cannot be empty".to_string()); + } + if self.cost_assumption.trim().is_empty() { + return Err("cost_assumption cannot be empty".to_string()); + } + if self.multiple_testing_control.trim().is_empty() { + return Err("multiple_testing_control cannot be empty".to_string()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BacktestData { + pub returns: Vec, + pub label_spans: Vec<(NaiveDateTime, NaiveDateTime)>, +} + +impl BacktestData { + pub fn validate(&self) -> Result<(), String> { + if self.returns.is_empty() { + return Err("returns cannot be empty".to_string()); + } + if self.returns.len() != self.label_spans.len() { + return Err("returns and label_spans length mismatch".to_string()); + } + if self.returns.iter().any(|r| !r.is_finite()) { + return Err("returns must be finite".to_string()); + } + for (start, end) in &self.label_spans { + if end < start { + return Err("label span end must be >= start".to_string()); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BacktestMode { + WalkForward, + CrossValidation, + CombinatorialPurgedCrossValidation, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SplitDefinition { + pub split_id: usize, + pub train_indices: Vec, + pub test_indices: Vec, + pub test_groups: Vec, + pub purged_count: usize, + pub embargo_count: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct FoldPerformance { + pub split_id: usize, + pub sharpe: f64, + pub mean_return: f64, + pub std_return: f64, + pub observations: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AntiLeakageDiagnostics { + pub uses_label_span_purging: bool, + pub uses_embargo: bool, + pub total_purged: usize, + pub total_embargoed: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BacktestDiagnostics { + pub mode: BacktestMode, + pub mode_provenance: String, + pub trials_count: usize, + pub split_count: usize, + pub pct_embargo: f64, + pub safeguards: BacktestSafeguards, + pub anti_leakage: AntiLeakageDiagnostics, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct WalkForwardConfig { + pub min_train_size: usize, + pub test_size: usize, + pub step_size: usize, + pub pct_embargo: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CrossValidationConfig { + pub n_splits: usize, + pub pct_embargo: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CpcvConfig { + pub n_groups: usize, + pub test_groups: usize, + pub pct_embargo: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct BacktestRunConfig { + pub mode_provenance: String, + pub trials_count: usize, + pub safeguards: BacktestSafeguards, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CpcvPathAssignment { + pub path_id: usize, + pub split_for_group: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CpcvPathPerformance { + pub path_id: usize, + pub sharpe: f64, + pub mean_return: f64, + pub std_return: f64, + pub observations: usize, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct WalkForwardResult { + pub folds: Vec, + pub splits: Vec, + pub diagnostics: BacktestDiagnostics, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CrossValidationResult { + pub folds: Vec, + pub splits: Vec, + pub diagnostics: BacktestDiagnostics, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CpcvResult { + pub folds: Vec, + pub splits: Vec, + pub path_count: usize, + pub path_assignments: Vec, + pub path_distribution: Vec, + pub diagnostics: BacktestDiagnostics, +} + +pub fn cpcv_path_count(n_groups: usize, test_groups: usize) -> Result { + validate_cpcv_params(n_groups, test_groups)?; + let total = n_choose_k(n_groups, test_groups)?; + Ok((total * test_groups) / n_groups) +} + +pub fn run_walk_forward( + data: &BacktestData, + run: &BacktestRunConfig, + config: &WalkForwardConfig, + mut evaluator: E, +) -> Result +where + E: FnMut(&SplitDefinition) -> Result, String>, +{ + data.validate()?; + run.validate(BacktestMode::WalkForward)?; + validate_embargo(config.pct_embargo)?; + if config.min_train_size == 0 { + return Err("min_train_size must be > 0".to_string()); + } + if config.test_size == 0 { + return Err("test_size must be > 0".to_string()); + } + if config.step_size == 0 { + return Err("step_size must be > 0".to_string()); + } + + let n_samples = data.returns.len(); + let mut split_defs = Vec::new(); + let mut start = config.min_train_size; + let mut split_id = 0; + + while start < n_samples { + let stop = (start + config.test_size).min(n_samples); + let test_indices: Vec = (start..stop).collect(); + if test_indices.is_empty() { + break; + } + let initial_train: Vec = (0..start).collect(); + let (train_indices, purged_count, embargo_count) = apply_purge_and_embargo( + &initial_train, + &test_indices, + &data.label_spans, + config.pct_embargo, + n_samples, + ); + if train_indices.is_empty() { + return Err("walk-forward produced an empty train split".to_string()); + } + split_defs.push(SplitDefinition { + split_id, + train_indices, + test_indices, + test_groups: vec![split_id], + purged_count, + embargo_count, + }); + split_id += 1; + start += config.step_size; + } + + if split_defs.is_empty() { + return Err("walk-forward produced no splits".to_string()); + } + + let folds = evaluate_splits(&split_defs, &mut evaluator)?; + let diagnostics = + build_diagnostics(BacktestMode::WalkForward, run, config.pct_embargo, &split_defs); + Ok(WalkForwardResult { folds, splits: split_defs, diagnostics }) +} + +pub fn run_cross_validation( + data: &BacktestData, + run: &BacktestRunConfig, + config: &CrossValidationConfig, + mut evaluator: E, +) -> Result +where + E: FnMut(&SplitDefinition) -> Result, String>, +{ + data.validate()?; + run.validate(BacktestMode::CrossValidation)?; + validate_embargo(config.pct_embargo)?; + if config.n_splits < 2 { + return Err("n_splits must be >= 2".to_string()); + } + + let base_test_splits = contiguous_folds(data.returns.len(), config.n_splits)?; + let mut split_defs = Vec::with_capacity(base_test_splits.len()); + for (split_id, test_indices) in base_test_splits.into_iter().enumerate() { + let initial_train: Vec = + (0..data.returns.len()).filter(|idx| !test_indices.contains(idx)).collect(); + let (train_indices, purged_count, embargo_count) = apply_purge_and_embargo( + &initial_train, + &test_indices, + &data.label_spans, + config.pct_embargo, + data.returns.len(), + ); + if train_indices.is_empty() { + return Err("cross-validation produced an empty train split".to_string()); + } + split_defs.push(SplitDefinition { + split_id, + train_indices, + test_indices, + test_groups: vec![split_id], + purged_count, + embargo_count, + }); + } + + let folds = evaluate_splits(&split_defs, &mut evaluator)?; + let diagnostics = + build_diagnostics(BacktestMode::CrossValidation, run, config.pct_embargo, &split_defs); + Ok(CrossValidationResult { folds, splits: split_defs, diagnostics }) +} + +pub fn run_cpcv( + data: &BacktestData, + run: &BacktestRunConfig, + config: &CpcvConfig, + mut evaluator: E, +) -> Result +where + E: FnMut(&SplitDefinition) -> Result, String>, +{ + data.validate()?; + run.validate(BacktestMode::CombinatorialPurgedCrossValidation)?; + validate_cpcv_params(config.n_groups, config.test_groups)?; + validate_embargo(config.pct_embargo)?; + + let group_index_map = contiguous_folds(data.returns.len(), config.n_groups)?; + let group_combinations = combinations(config.n_groups, config.test_groups); + let mut split_defs = Vec::with_capacity(group_combinations.len()); + + for (split_id, groups) in group_combinations.iter().enumerate() { + let mut test_indices = Vec::new(); + for g in groups { + test_indices.extend(group_index_map[*g].iter().copied()); + } + test_indices.sort_unstable(); + + let initial_train: Vec = + (0..data.returns.len()).filter(|idx| !test_indices.contains(idx)).collect(); + let (train_indices, purged_count, embargo_count) = apply_purge_and_embargo( + &initial_train, + &test_indices, + &data.label_spans, + config.pct_embargo, + data.returns.len(), + ); + + if train_indices.is_empty() { + return Err("CPCV produced an empty train split".to_string()); + } + + split_defs.push(SplitDefinition { + split_id, + train_indices, + test_indices, + test_groups: groups.clone(), + purged_count, + embargo_count, + }); + } + + let (folds, split_returns) = evaluate_splits_with_returns(&split_defs, &mut evaluator)?; + let path_count = cpcv_path_count(config.n_groups, config.test_groups)?; + let path_assignments = build_cpcv_path_assignments(config.n_groups, &split_defs, path_count)?; + let path_distribution = build_path_distribution( + config.n_groups, + &group_index_map, + &path_assignments, + &split_defs, + &split_returns, + )?; + + let diagnostics = build_diagnostics( + BacktestMode::CombinatorialPurgedCrossValidation, + run, + config.pct_embargo, + &split_defs, + ); + + Ok(CpcvResult { + folds, + splits: split_defs, + path_count, + path_assignments, + path_distribution, + diagnostics, + }) +} + +fn evaluate_splits( + splits: &[SplitDefinition], + evaluator: &mut E, +) -> Result, String> +where + E: FnMut(&SplitDefinition) -> Result, String>, +{ + let mut out = Vec::with_capacity(splits.len()); + for split in splits { + let split_returns = evaluator(split)?; + let perf = summarize_returns(split.split_id, &split_returns)?; + out.push(perf); + } + Ok(out) +} + +fn evaluate_splits_with_returns( + splits: &[SplitDefinition], + evaluator: &mut E, +) -> Result<(Vec, HashMap>), String> +where + E: FnMut(&SplitDefinition) -> Result, String>, +{ + let mut out = Vec::with_capacity(splits.len()); + let mut split_returns = HashMap::with_capacity(splits.len()); + for split in splits { + let returns = evaluator(split)?; + let perf = summarize_returns(split.split_id, &returns)?; + out.push(perf); + split_returns.insert(split.split_id, returns); + } + Ok((out, split_returns)) +} + +fn summarize_returns(split_id: usize, returns: &[f64]) -> Result { + if returns.is_empty() { + return Err("split evaluator returned empty returns".to_string()); + } + if returns.iter().any(|r| !r.is_finite()) { + return Err("split evaluator returned non-finite returns".to_string()); + } + let n = returns.len(); + let mean = returns.iter().sum::() / n as f64; + let variance = if n > 1 { + returns.iter().map(|r| (r - mean).powi(2)).sum::() / (n as f64 - 1.0) + } else { + 0.0 + }; + let std = variance.sqrt(); + let sharpe = if std > 0.0 { mean / std * (n as f64).sqrt() } else { 0.0 }; + Ok(FoldPerformance { split_id, sharpe, mean_return: mean, std_return: std, observations: n }) +} + +fn build_diagnostics( + mode: BacktestMode, + run: &BacktestRunConfig, + pct_embargo: f64, + splits: &[SplitDefinition], +) -> BacktestDiagnostics { + let total_purged = splits.iter().map(|s| s.purged_count).sum::(); + let total_embargoed = splits.iter().map(|s| s.embargo_count).sum::(); + BacktestDiagnostics { + mode, + mode_provenance: run.mode_provenance.clone(), + trials_count: run.trials_count, + split_count: splits.len(), + pct_embargo, + safeguards: run.safeguards.clone(), + anti_leakage: AntiLeakageDiagnostics { + uses_label_span_purging: true, + uses_embargo: pct_embargo > 0.0, + total_purged, + total_embargoed, + }, + } +} + +impl BacktestRunConfig { + fn validate(&self, mode: BacktestMode) -> Result<(), String> { + if self.mode_provenance.trim().is_empty() { + return Err("mode_provenance cannot be empty".to_string()); + } + if self.trials_count == 0 { + return Err("trials_count must be > 0".to_string()); + } + self.safeguards.validate()?; + match mode { + BacktestMode::WalkForward + | BacktestMode::CrossValidation + | BacktestMode::CombinatorialPurgedCrossValidation => Ok(()), + } + } +} + +fn validate_embargo(pct_embargo: f64) -> Result<(), String> { + if !(0.0..1.0).contains(&pct_embargo) { + return Err("pct_embargo must be in [0,1)".to_string()); + } + Ok(()) +} + +fn validate_cpcv_params(n_groups: usize, test_groups: usize) -> Result<(), String> { + if n_groups < 2 { + return Err("n_groups must be >= 2".to_string()); + } + if test_groups == 0 { + return Err("test_groups must be > 0".to_string()); + } + if test_groups >= n_groups { + return Err("test_groups must be < n_groups".to_string()); + } + Ok(()) +} + +fn contiguous_folds(n_samples: usize, n_folds: usize) -> Result>, String> { + if n_folds == 0 { + return Err("n_folds must be > 0".to_string()); + } + if n_folds > n_samples { + return Err("n_folds cannot exceed number of samples".to_string()); + } + + let mut fold_sizes = vec![n_samples / n_folds; n_folds]; + for size in fold_sizes.iter_mut().take(n_samples % n_folds) { + *size += 1; + } + + let mut current = 0; + let mut out = Vec::with_capacity(n_folds); + for fold_size in fold_sizes { + let indices: Vec = (current..(current + fold_size)).collect(); + out.push(indices); + current += fold_size; + } + Ok(out) +} + +fn overlaps(lhs: (NaiveDateTime, NaiveDateTime), rhs: (NaiveDateTime, NaiveDateTime)) -> bool { + lhs.0 <= rhs.1 && rhs.0 <= lhs.1 +} + +fn apply_purge_and_embargo( + initial_train: &[usize], + test_indices: &[usize], + label_spans: &[(NaiveDateTime, NaiveDateTime)], + pct_embargo: f64, + n_samples: usize, +) -> (Vec, usize, usize) { + let mut train_mask = vec![false; n_samples]; + let mut initial_train_mask = vec![false; n_samples]; + for idx in initial_train { + train_mask[*idx] = true; + initial_train_mask[*idx] = true; + } + + let mut purged_count = 0; + for idx in initial_train { + let mut should_purge = false; + for test_idx in test_indices { + if overlaps(label_spans[*idx], label_spans[*test_idx]) { + should_purge = true; + break; + } + } + if should_purge { + train_mask[*idx] = false; + purged_count += 1; + } + } + + let embargo_width = (pct_embargo * n_samples as f64).ceil() as usize; + let mut embargoed = vec![false; n_samples]; + if embargo_width > 0 { + for test_idx in test_indices { + let start = test_idx.saturating_sub(embargo_width); + let stop = (*test_idx + embargo_width + 1).min(n_samples); + for idx in start..stop { + if initial_train_mask[idx] { + embargoed[idx] = true; + } + } + } + for idx in 0..n_samples { + if embargoed[idx] { + train_mask[idx] = false; + } + } + } + + let train_indices: Vec = train_mask + .iter() + .enumerate() + .filter_map(|(idx, keep)| if *keep { Some(idx) } else { None }) + .collect(); + let embargo_count = embargoed.into_iter().filter(|v| *v).count(); + + (train_indices, purged_count, embargo_count) +} + +fn n_choose_k(n: usize, k: usize) -> Result { + if k > n { + return Err("k cannot exceed n".to_string()); + } + let k_eff = k.min(n - k); + let mut numerator: u128 = 1; + let mut denominator: u128 = 1; + for i in 0..k_eff { + numerator *= (n - i) as u128; + denominator *= (i + 1) as u128; + } + let comb = numerator / denominator; + usize::try_from(comb).map_err(|_| "combination count overflowed usize".to_string()) +} + +fn combinations(n: usize, k: usize) -> Vec> { + let mut out = Vec::new(); + let mut current = Vec::with_capacity(k); + combinations_recursive(0, n, k, &mut current, &mut out); + out +} + +fn combinations_recursive( + start: usize, + n: usize, + k: usize, + current: &mut Vec, + out: &mut Vec>, +) { + if current.len() == k { + out.push(current.clone()); + return; + } + for i in start..n { + current.push(i); + combinations_recursive(i + 1, n, k, current, out); + current.pop(); + } +} + +fn build_cpcv_path_assignments( + n_groups: usize, + splits: &[SplitDefinition], + path_count: usize, +) -> Result, String> { + let mut group_occurrences: Vec> = vec![Vec::new(); n_groups]; + for split in splits { + for g in &split.test_groups { + if *g >= n_groups { + return Err("split references out-of-range test group".to_string()); + } + group_occurrences[*g].push(split.split_id); + } + } + + for (group_idx, occurrences) in group_occurrences.iter().enumerate() { + if occurrences.len() != path_count { + return Err(format!( + "group {group_idx} has {} occurrences, expected {path_count}", + occurrences.len() + )); + } + } + + let mut assignments = Vec::with_capacity(path_count); + for path_id in 0..path_count { + let mut split_for_group = Vec::with_capacity(n_groups); + for occurrences in group_occurrences.iter().take(n_groups) { + split_for_group.push(occurrences[path_id]); + } + assignments.push(CpcvPathAssignment { path_id, split_for_group }); + } + Ok(assignments) +} + +fn build_path_distribution( + n_groups: usize, + group_index_map: &[Vec], + assignments: &[CpcvPathAssignment], + splits: &[SplitDefinition], + split_returns: &HashMap>, +) -> Result, String> { + let mut out = Vec::with_capacity(assignments.len()); + for assignment in assignments { + if assignment.split_for_group.len() != n_groups { + return Err("invalid path assignment length".to_string()); + } + + let mut path_returns = Vec::new(); + for (group_id, split_id) in assignment.split_for_group.iter().enumerate() { + let split = splits + .iter() + .find(|s| s.split_id == *split_id) + .ok_or_else(|| "path references unknown split".to_string())?; + if !split.test_groups.contains(&group_id) { + return Err("path assignment references split not containing group".to_string()); + } + let split_path_returns = split_returns + .get(split_id) + .ok_or_else(|| "missing split returns for CPCV path construction".to_string())?; + if split_path_returns.len() != split.test_indices.len() { + return Err("split return count must match split test indices length".to_string()); + } + let return_by_index: HashMap = split + .test_indices + .iter() + .copied() + .zip(split_path_returns.iter().copied()) + .collect(); + + for idx in &group_index_map[group_id] { + if split.test_indices.contains(idx) { + let r = return_by_index + .get(idx) + .ok_or_else(|| "split returns missing group test index".to_string())?; + path_returns.push(*r); + } + } + } + + let perf = summarize_returns(assignment.path_id, &path_returns)?; + out.push(CpcvPathPerformance { + path_id: assignment.path_id, + sharpe: perf.sharpe, + mean_return: perf.mean_return, + std_return: perf.std_return, + observations: perf.observations, + }); + } + Ok(out) +} diff --git a/crates/openquant/src/lib.rs b/crates/openquant/src/lib.rs index 568ac8b..4b06d6e 100644 --- a/crates/openquant/src/lib.rs +++ b/crates/openquant/src/lib.rs @@ -1,4 +1,5 @@ pub mod backtest_statistics; +pub mod backtesting_engine; pub mod bet_sizing; pub mod cla; pub mod codependence; diff --git a/crates/openquant/tests/backtesting_engine.rs b/crates/openquant/tests/backtesting_engine.rs new file mode 100644 index 0000000..663a434 --- /dev/null +++ b/crates/openquant/tests/backtesting_engine.rs @@ -0,0 +1,147 @@ +use chrono::{Duration, NaiveDateTime}; +use openquant::backtesting_engine::{ + cpcv_path_count, run_cpcv, run_cross_validation, run_walk_forward, BacktestData, BacktestMode, + BacktestRunConfig, BacktestSafeguards, CpcvConfig, CrossValidationConfig, WalkForwardConfig, +}; + +fn build_data(n: usize) -> BacktestData { + let start = NaiveDateTime::parse_from_str("2024-01-01 09:30:00", "%Y-%m-%d %H:%M:%S") + .expect("valid start timestamp"); + let mut returns = Vec::with_capacity(n); + let mut spans = Vec::with_capacity(n); + for i in 0..n { + let t0 = start + Duration::minutes(i as i64); + // Intentionally overlapping spans (3-minute horizon) to force purge logic. + let t1 = t0 + Duration::minutes(3); + spans.push((t0, t1)); + let sign = if i % 3 == 0 { -1.0 } else { 1.0 }; + returns.push(sign * (0.001 + i as f64 * 0.0002)); + } + BacktestData { returns, label_spans: spans } +} + +fn run_cfg(mode: BacktestMode) -> BacktestRunConfig { + BacktestRunConfig { + mode_provenance: format!("afml_ch11_ch12_{mode:?}"), + trials_count: 19, + safeguards: BacktestSafeguards { + survivorship_bias_control: "Universe frozen at decision timestamp".to_string(), + look_ahead_control: "Features lagged and label horizon aligned".to_string(), + data_mining_control: "Validation chosen ex-ante and logged".to_string(), + cost_assumption: "Linear + spread impact model applied".to_string(), + multiple_testing_control: "Trials count tracked for post-selection stats".to_string(), + }, + } +} + +fn spans_overlap(lhs: (NaiveDateTime, NaiveDateTime), rhs: (NaiveDateTime, NaiveDateTime)) -> bool { + lhs.0 <= rhs.1 && rhs.0 <= lhs.1 +} + +#[test] +fn cpcv_path_count_matches_phi_formula() { + let phi_6_2 = cpcv_path_count(6, 2).expect("valid CPCV parameters"); + let phi_8_3 = cpcv_path_count(8, 3).expect("valid CPCV parameters"); + assert_eq!(phi_6_2, 5); // C(6,2)*2/6 = 5 + assert_eq!(phi_8_3, 21); // C(8,3)*3/8 = 21 +} + +#[test] +fn walk_forward_and_cv_produce_explicit_outputs_and_diagnostics() { + let data = build_data(24); + + let wf = run_walk_forward( + &data, + &run_cfg(BacktestMode::WalkForward), + &WalkForwardConfig { min_train_size: 10, test_size: 4, step_size: 4, pct_embargo: 0.05 }, + |split| { + Ok(split + .test_indices + .iter() + .map(|idx| data.returns[*idx] + split.train_indices.len() as f64 * 1e-5) + .collect()) + }, + ) + .expect("walk-forward run should succeed"); + + assert!(!wf.folds.is_empty()); + assert_eq!(wf.folds.len(), wf.splits.len()); + assert_eq!(wf.diagnostics.mode, BacktestMode::WalkForward); + assert_eq!(wf.diagnostics.trials_count, 19); + assert!(wf.diagnostics.anti_leakage.total_purged > 0); + + let cv = run_cross_validation( + &data, + &run_cfg(BacktestMode::CrossValidation), + &CrossValidationConfig { n_splits: 4, pct_embargo: 0.05 }, + |split| { + Ok(split + .test_indices + .iter() + .map(|idx| data.returns[*idx] - split.split_id as f64 * 5e-6) + .collect()) + }, + ) + .expect("cross-validation run should succeed"); + + assert_eq!(cv.folds.len(), 4); + assert_eq!(cv.diagnostics.mode, BacktestMode::CrossValidation); + assert_eq!(cv.diagnostics.mode_provenance, "afml_ch11_ch12_CrossValidation"); +} + +#[test] +fn cpcv_enforces_purge_embargo_and_returns_path_distribution() { + let data = build_data(30); + let cfg = CpcvConfig { n_groups: 6, test_groups: 2, pct_embargo: 0.1 }; + + let result = run_cpcv( + &data, + &run_cfg(BacktestMode::CombinatorialPurgedCrossValidation), + &cfg, + |split| { + Ok(split + .test_indices + .iter() + .enumerate() + .map(|(j, idx)| { + // Slight split-dependent perturbation to produce non-degenerate path distribution. + data.returns[*idx] + split.split_id as f64 * 2e-5 + j as f64 * 1e-6 + }) + .collect()) + }, + ) + .expect("CPCV run should succeed"); + + let expected_phi = cpcv_path_count(cfg.n_groups, cfg.test_groups).expect("valid phi"); + assert_eq!(result.path_count, expected_phi); + assert_eq!(result.path_assignments.len(), expected_phi); + assert_eq!(result.path_distribution.len(), expected_phi); + assert_eq!(result.folds.len(), result.splits.len()); + assert_eq!(result.diagnostics.mode, BacktestMode::CombinatorialPurgedCrossValidation); + assert_eq!(result.diagnostics.split_count, result.splits.len()); + + let mut observed_sharpes = + result.path_distribution.iter().map(|p| p.sharpe).collect::>(); + observed_sharpes.sort_by(|a, b| a.partial_cmp(b).expect("sharpe values should be finite")); + observed_sharpes.dedup_by(|a, b| (*a - *b).abs() < 1e-12); + assert!( + observed_sharpes.len() >= 2, + "path distribution should not collapse to one point estimate" + ); + + for split in &result.splits { + for train_idx in &split.train_indices { + for test_idx in &split.test_indices { + let overlaps = + spans_overlap(data.label_spans[*train_idx], data.label_spans[*test_idx]); + assert!( + !overlaps, + "train index {train_idx} should be purged when overlapping test index {test_idx}" + ); + } + } + } + + assert!(result.diagnostics.anti_leakage.total_purged > 0); + assert!(result.diagnostics.anti_leakage.total_embargoed > 0); +} diff --git a/docs-site/src/data/afmlDocsState.ts b/docs-site/src/data/afmlDocsState.ts index 1e39a90..7de54ec 100644 --- a/docs-site/src/data/afmlDocsState.ts +++ b/docs-site/src/data/afmlDocsState.ts @@ -170,6 +170,34 @@ export const afmlDocsState = { } ] }, + { + "chapter": "CHAPTER 11", + "theme": "Backtest logic and pitfalls", + "status": "done", + "chunkCount": 24, + "sections": [ + { + "id": "chapter-11-backtesting_engine", + "module": "backtesting_engine", + "slug": "backtesting-engine", + "status": "done" + } + ] + }, + { + "chapter": "CHAPTER 12", + "theme": "Walk-forward and CPCV", + "status": "done", + "chunkCount": 28, + "sections": [ + { + "id": "chapter-12-backtesting_engine", + "module": "backtesting_engine", + "slug": "backtesting-engine", + "status": "done" + } + ] + }, { "chapter": "CHAPTER 14", "theme": "Backtest diagnostics", diff --git a/docs-site/src/data/moduleDocs.ts b/docs-site/src/data/moduleDocs.ts index b6dc7d9..a9c8bd7 100644 --- a/docs-site/src/data/moduleDocs.ts +++ b/docs-site/src/data/moduleDocs.ts @@ -51,6 +51,52 @@ export const moduleDocs: ModuleDoc[] = [ "Deflated Sharpe is useful when strategy mining many variants.", ], }, + { + slug: "backtesting-engine", + module: "backtesting_engine", + subject: "Sampling, Validation and ML Diagnostics", + summary: "Backtesting core with walk-forward, purged CV, and combinatorial purged CV (CPCV) workflows.", + whyItExists: + "AFML Chapters 11-12 require scenario-based validation with explicit anti-leakage controls, split provenance, and path-wise uncertainty rather than single-score reporting.", + keyApis: [ + "run_walk_forward", + "run_cross_validation", + "run_cpcv", + "cpcv_path_count", + "BacktestRunConfig", + "BacktestSafeguards", + "WalkForwardConfig", + "CrossValidationConfig", + "CpcvConfig", + ], + formulas: [ + { + label: "CPCV Path Count", + latex: "\\phi[N,k]=\\binom{N}{k}\\frac{k}{N}=\\binom{N-1}{k-1}", + }, + { + label: "Purge + Embargo Train Set", + latex: + "\\mathcal T_{train}^{*}=\\mathcal T_{train}\\setminus\\{i: \\exists j\\in\\mathcal T_{test},\\;I_i\\cap I_j\\neq\\varnothing\\}\\setminus\\mathcal E(\\mathcal T_{test},p)", + }, + { + label: "Per-Path Sharpe", + latex: "S_{path}=\\frac{\\bar r_{path}}{\\sigma_{path}}\\sqrt{T_{path}}", + }, + ], + examples: [ + { + title: "Run CPCV and inspect Sharpe distribution", + language: "rust", + code: `use openquant::backtesting_engine::{\n run_cpcv, BacktestData, BacktestRunConfig, BacktestSafeguards, CpcvConfig,\n};\n\nlet result = run_cpcv(\n &data,\n &BacktestRunConfig {\n mode_provenance: \"research_v3_with_costs\".to_string(),\n trials_count: 24,\n safeguards: BacktestSafeguards {\n survivorship_bias_control: \"point-in-time universe\".to_string(),\n look_ahead_control: \"lagged features\".to_string(),\n data_mining_control: \"frozen split protocol\".to_string(),\n cost_assumption: \"spread + slippage\".to_string(),\n multiple_testing_control: \"trial count logged\".to_string(),\n },\n },\n &CpcvConfig { n_groups: 8, test_groups: 2, pct_embargo: 0.01 },\n |split| Ok(split.test_indices.iter().map(|i| pnl[*i]).collect()),\n)?;\n\nprintln!(\"phi = {}\", result.path_count);\nprintln!(\"path sharpe count = {}\", result.path_distribution.len());`, + }, + ], + notes: [ + "Chapter 11: a backtest is a scenario sanity check; keep safeguards and assumptions attached to every run.", + "Chapter 12: compare WF/CV/CPCV results by mode rather than averaging them into one statistic.", + "CPCV output is a path distribution, enabling robust Sharpe diagnostics (e.g., quantiles) instead of point estimates.", + ], + }, { slug: "bet-sizing", module: "bet_sizing", diff --git a/docs-site/src/pages/api-reference.astro b/docs-site/src/pages/api-reference.astro index bbba6db..638e850 100644 --- a/docs-site/src/pages/api-reference.astro +++ b/docs-site/src/pages/api-reference.astro @@ -19,6 +19,7 @@ import Layout from "../layouts/Layout.astro";
  • sampling::seq_bootstrap, sampling::get_ind_matrix, sampling::get_ind_mat_average_uniqueness
  • cross_validation::ml_cross_val_score, cross_validation::ml_get_train_times, cross_validation::PurgedKFold
    • +
  • backtesting_engine::run_walk_forward, backtesting_engine::run_cross_validation, backtesting_engine::run_cpcv, backtesting_engine::cpcv_path_count
  • hyperparameter_tuning::grid_search, hyperparameter_tuning::randomized_search, hyperparameter_tuning::sample_log_uniform, hyperparameter_tuning::classification_score
  • feature_importance::mean_decrease_impurity, feature_importance::mean_decrease_accuracy, feature_importance::single_feature_importance
  • fingerprint::RegressionModelFingerprint, fingerprint::ClassificationModelFingerprint