From d34e6013e4c616f2d2ed7219686c2bcc7ffa1eec Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Thu, 24 Sep 2026 12:43:37 +0100 Subject: [PATCH 01/18] Add a pairing filter and selection accounting to ToggleSpecialistMethod - pairing_max_gap: keep a used row only if the other segment has a used row within the gap, inclusive. Must be a positive whole multiple of the timebase, so a sub-timebase value can no longer silently drop every row (v0's floored-seconds semantics). Applied after exclude_row, so the uplift and the bootstrap see the same selection. Parity-tested against v0's any_within_timedelta. - MethodOutput.selection_accounting: rows kept per segment after each stage (segment, filters, exclude_row, pairing), also written to a selection CSV; a warning when a stage keeps the two segments' rows unevenly (segment_imbalance_warning). - Pairing-gap diagnostic plot (2_filter): each used row's gap to the nearest used row of the other segment, before and after pairing, with hours kept. Drawn without a pairing filter too, to judge whether one is needed. - Path-separator-agnostic assertion in test_uplift_plots (Windows fix, as on northing-improvement). Co-Authored-By: Claude Opus 5.5 --- benchmarking/baselines/toggle_specialist.py | 248 +++++++++++++++++- benchmarking/harness/method.py | 5 + .../baselines/test_toggle_specialist.py | 153 ++++++++++- 3 files changed, 395 insertions(+), 11 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 09544eb6..d3402010 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -12,6 +12,11 @@ never conditions on the test turbine's post-treatment wind speed (design-note §3). It speaks the data source's own column names and has no wind_up dependency. +An optional **pairing filter** (``pairing_max_gap``) then drops any used row with no used row of the +other segment nearby, so a filter that removes one segment's rows in some conditions also removes the +other segment's rows from those conditions. Rows kept per segment after each selection stage are +reported as ``MethodOutput.selection_accounting`` and written to a selection CSV. + Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the uplift, from the uplift's own frozen row selection and bin assignment, and only when the uplift is @@ -28,10 +33,11 @@ from __future__ import annotations +import logging import tempfile from dataclasses import dataclass, replace from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import matplotlib.pyplot as plt import numpy as np @@ -69,6 +75,57 @@ # post-treatment signals, which this method exists not to do (see the module docstring). _SUPPORTED_CONDITIONS: tuple[str, ...] = ("power",) +logger = logging.getLogger(__name__) + + +class _Selection(NamedTuple): + """The cumulative used-row mask after each selection stage.""" + + filtered: pd.Series + not_excluded: pd.Series + paired: pd.Series + + +class PairedRows(NamedTuple): + """The rows of each segment that have a row of the other segment within the pairing gap.""" + + baseline: npt.NDArray[np.bool_] + upgraded: npt.NDArray[np.bool_] + + +def pair_within( + index: pd.DatetimeIndex, + *, + baseline: npt.NDArray[np.bool_], + upgraded: npt.NDArray[np.bool_], + max_gap: pd.Timedelta, +) -> PairedRows: + """Keep each flagged row only if the other segment has a flagged row within ``max_gap``, inclusive.""" + gap = gap_to_other_segment(index, baseline=baseline, upgraded=upgraded) + within = gap <= max_gap.total_seconds() + return PairedRows(baseline=baseline & within, upgraded=upgraded & within) + + +def gap_to_other_segment( + index: pd.DatetimeIndex, *, baseline: npt.NDArray[np.bool_], upgraded: npt.NDArray[np.bool_] +) -> npt.NDArray[np.float64]: + """Seconds from each flagged row to the nearest flagged row of the other segment; NaN if unflagged.""" + times = index.asi8 + gap = np.full(len(index), np.nan) + gap[baseline] = _nearest_gap_s(times[baseline], others=times[upgraded]) + gap[upgraded] = _nearest_gap_s(times[upgraded], others=times[baseline]) + return gap + + +def _nearest_gap_s(times: npt.NDArray[np.int64], *, others: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]: + if not len(others): + return np.full(len(times), np.inf) + others = np.sort(others) + position = np.searchsorted(others, times) + after = others[np.minimum(position, len(others) - 1)] + before = others[np.maximum(position - 1, 0)] + return np.minimum(np.abs(after - times), np.abs(times - before)) / 1e9 + def _infer_timebase(index: pd.DatetimeIndex) -> pd.Timedelta: """Infer the analysis timebase as the median spacing of the sorted unique timestamps.""" @@ -127,6 +184,11 @@ class ToggleSpecialistMethod: period**, for which :data:`DEFAULT_BLOCK_HOURS` may span only a cycle or two. :param n_resamples: bootstrap resamples; block sums are precomputed, so this can be generous. :param bootstrap_seed: RNG seed for the bootstrap, so a reported sigma is reproducible. + :param pairing_max_gap: when set, a used row of one segment is kept only if the other segment has + a used row within this gap, inclusive (a gap of exactly ``pairing_max_gap`` is kept). Must be + a positive whole multiple of the timebase; ``None`` disables pairing. + :param segment_imbalance_warning: warn when the two segments' kept fractions at any selection + stage differ by more than this. """ columns: ColumnSchema @@ -139,6 +201,8 @@ class ToggleSpecialistMethod: block_hours: float = DEFAULT_BLOCK_HOURS n_resamples: int = 1000 bootstrap_seed: int = 0 + pairing_max_gap: pd.Timedelta | None = None + segment_imbalance_warning: float = 0.1 def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" @@ -184,11 +248,14 @@ def estimate(self, mi: MethodInput) -> MethodOutput: raise ValueError(msg) timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) + self._check_pairing_gap(timebase) rows = resolve_toggle(mi.upgrade_timing, wide.index) baseline = rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) ref_total = wide[refs].sum(axis=1).to_numpy(dtype=float) - used = self._used_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase).to_numpy() + selection = self._selection(mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows) + used = selection.paired.to_numpy() + accounting = self._selection_accounting(selection, rows=rows) rho_base = _rho(test_pw, ref_total, used & baseline) rho_up = _rho(test_pw, ref_total, used & rows.upgraded) @@ -258,6 +325,9 @@ def estimate(self, mi: MethodInput) -> MethodOutput: timebase=timebase, per_bin=per_bin, diagnostics=diagnostics, + accounting=accounting, + selection=selection, + rows=rows, ) return MethodOutput( p50_overall=float(uplift), @@ -267,6 +337,7 @@ def estimate(self, mi: MethodInput) -> MethodOutput: labeled_rows=self._labeled_rows( mi, wide=wide, test=test, used=used, rows=rows, rho_label=rho_label, ref_total=ref_total ), + selection_accounting=accounting, ) def _labeled_rows( @@ -396,7 +467,90 @@ def _conditional_frame( frame.insert(0, "condition", "power") return frame - def _used_mask( + def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: + gap = self.pairing_max_gap + if gap is None: + return + if gap <= pd.Timedelta(0) or gap % timebase != pd.Timedelta(0): + msg = ( + f"{self.name}: pairing_max_gap {gap} must be a positive whole multiple of the timebase " + f"{timebase}; a shorter gap can never reach a row of the other segment." + ) + raise ValueError(msg) + + def _selection( + self, + mi: MethodInput, + *, + wide: pd.DataFrame, + test: str, + refs: list[str], + timebase: pd.Timedelta, + rows: ToggleRowSets, + ) -> _Selection: + """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``.""" + filtered = self._filtered_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase) + not_excluded = filtered & ~self._test_excluded(mi, test=test, index=wide.index) + if self.pairing_max_gap is None: + return _Selection(filtered=filtered, not_excluded=not_excluded, paired=not_excluded) + kept = not_excluded.to_numpy() + paired = pair_within( + wide.index, + baseline=kept & rows.campaign_baseline, + upgraded=kept & rows.upgraded, + max_gap=self.pairing_max_gap, + ) + in_segment = rows.campaign_baseline | rows.upgraded + paired_mask = kept & (~in_segment | paired.baseline | paired.upgraded) + return _Selection(filtered=filtered, not_excluded=not_excluded, paired=pd.Series(paired_mask, index=wide.index)) + + def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) -> pd.DataFrame: + """Rows kept per segment after each stage, warning when a stage treats the segments unevenly. + + ``kept_fraction`` is relative to the segment's rows; ``stage_kept_fraction`` to the previous + stage, so it isolates which stage discriminates between segments. + """ + stages_masks = ( + ("segment", None), + ("filters", selection.filtered), + ("exclude_row", selection.not_excluded), + ("pairing", selection.paired), + ) + records = [] + for segment, in_segment in ((_BASELINE, rows.campaign_baseline), (_UPGRADED, rows.upgraded)): + n_segment = int(in_segment.sum()) + previous = n_segment + for stage, mask in stages_masks: + n_kept = n_segment if mask is None else int((mask.to_numpy() & in_segment).sum()) + records.append( + { + "stage": stage, + "segment": segment, + "n_kept": n_kept, + "kept_fraction": n_kept / n_segment if n_segment else np.nan, + "stage_kept_fraction": n_kept / previous if previous else np.nan, + } + ) + previous = n_kept + accounting = pd.DataFrame(records) + + by_stage = accounting.pivot( # noqa: PD010 - one row per (stage, segment), nothing to aggregate + index="stage", columns="segment", values="stage_kept_fraction" + ) + imbalance = (by_stage[_BASELINE] - by_stage[_UPGRADED]).abs() + for stage, gap in imbalance[imbalance > self.segment_imbalance_warning].items(): + logger.warning( + "%s: stage %r kept %.1f%% of baseline vs %.1f%% of upgraded rows (difference %.1f%% > %.1f%%)", + self.name, + stage, + 100 * by_stage.loc[stage, _BASELINE], + 100 * by_stage.loc[stage, _UPGRADED], + 100 * gap, + 100 * self.segment_imbalance_warning, + ) + return accounting + + def _filtered_mask( self, mi: MethodInput, *, wide: pd.DataFrame, test: str, refs: list[str], timebase: pd.Timedelta ) -> pd.Series: """Complete-case timestamps that also pass downtime filtering on the test turbine and every reference. @@ -426,7 +580,7 @@ def _used_mask( .keep_mask(test_rows, timebase=timebase) .reindex(wide.index, fill_value=False) ) - return complete & all_available & test_keep & ~self._test_excluded(mi, test=test, index=wide.index) + return complete & all_available & test_keep def _test_excluded(self, mi: MethodInput, *, test: str, index: pd.DatetimeIndex) -> pd.Series: """Boolean mask on *index*: the test turbine's caller-flagged rows to drop (empty when unused). @@ -465,6 +619,9 @@ def _write_outputs( timebase: pd.Timedelta, per_bin: pd.DataFrame | None = None, diagnostics: pd.DataFrame | None = None, + accounting: pd.DataFrame | None = None, + selection: _Selection | None = None, + rows: ToggleRowSets | None = None, ) -> None: """Write the data-stats CSV, the headline results CSV, the per-bin CSV and (optionally) the plots.""" upgrade_start = toggle_upgrade_start(mi.upgrade_timing, wide.index) @@ -506,6 +663,8 @@ def _write_outputs( per_bin.to_csv(run_dir / f"{run_name}_by_power_bin_{ts}.csv", index=False) if diagnostics is not None: diagnostics.to_csv(run_dir / f"{run_name}_uncertainty_{ts}.csv", index=False) + if accounting is not None: + accounting.to_csv(run_dir / f"{run_name}_selection_{ts}.csv", index=False) if self.save_plots: _save_plots( @@ -524,18 +683,27 @@ def _write_outputs( test=mi.test_wtg, active_power_col=self.columns.active_power, ) - self._write_shared_diagnostics(mi, run_dir=run_dir, wide=wide, timebase=timebase) + if selection is not None and rows is not None: + _save_pairing_gap_plot( + run_dir / "plots" / stages.FILTER / f"{mi.test_wtg}_pairing_gap.png", + index=wide.index, + selection=selection, + rows=rows, + timebase=timebase, + max_gap=self.pairing_max_gap, + test=mi.test_wtg, + ) + self._write_shared_diagnostics(mi, run_dir=run_dir, wide=wide, used=used, timebase=timebase) def _write_shared_diagnostics( - self, mi: MethodInput, *, run_dir: Path, wide: pd.DataFrame, timebase: pd.Timedelta + self, mi: MethodInput, *, run_dir: Path, wide: pd.DataFrame, used: np.ndarray, timebase: pd.Timedelta ) -> None: """Emit the shared cross-method diagnostics (coverage/curves/histograms) and the run config.""" # ``wide`` (a pivot) drops all-NaN timestamps, so align the masks to the full unique index # the DiagnosticContext uses (timestamps absent from ``wide`` are simply not used). index = pd.DatetimeIndex(pd.unique(mi.scada_df.index)).sort_values() - test, refs = mi.test_wtg, [c for c in wide.columns if c != mi.test_wtg] - used_series = self._used_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase) - used = used_series.reindex(index, fill_value=False).to_numpy() + test = mi.test_wtg + used = pd.Series(used, index=wide.index).reindex(index, fill_value=False).to_numpy() treated = resolve_toggle(mi.upgrade_timing, index).upgraded.astype(bool) ctx = DiagnosticContext( run_dir=run_dir, @@ -555,6 +723,7 @@ def _write_shared_diagnostics( params = { "active_power_col": self.columns.active_power, "availability_col": self.columns.availability, + "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), } write_run_config(ctx, method_name=self.name, method_params=params) @@ -841,6 +1010,67 @@ def _save_plots( _save(fig, plots_dir / stages.FILTER / f"{test}_coverage_timeseries.png") +def _save_pairing_gap_plot( + path: Path, + *, + index: pd.DatetimeIndex, + selection: _Selection, + rows: ToggleRowSets, + timebase: pd.Timedelta, + max_gap: pd.Timedelta | None, + test: str, +) -> None: + """Histogram each used row's gap to the nearest used row of the other segment, before and after pairing. + + Gaps beyond the plotted range are piled into the last bin. + """ + step_min = timebase.total_seconds() / 60.0 + cap_steps = max(24, 3 * round(max_gap / timebase)) if max_gap is not None else 24 + edges = (np.arange(cap_steps + 2) - 0.5) * step_min + hours_per_row = timebase / pd.Timedelta(hours=1) + + fig, ax = plt.subplots(figsize=(9, 5)) + text = [] + drawn = [("before pairing" if max_gap is not None else "used rows", selection.not_excluded, "C0")] + if max_gap is not None: + drawn.append(("after pairing", selection.paired, "C1")) + for label, mask, color in drawn: + kept = mask.to_numpy() + gap_min = ( + gap_to_other_segment(index, baseline=kept & rows.campaign_baseline, upgraded=kept & rows.upgraded) / 60.0 + ) + gap_min = gap_min[~np.isnan(gap_min)] + ax.hist( + np.minimum(gap_min, cap_steps * step_min), + bins=edges, + histtype="stepfilled", + alpha=0.45, + color=color, + label=label, + ) + n_base = int((kept & rows.campaign_baseline).sum()) + n_up = int((kept & rows.upgraded).sum()) + text.append(f"{label}: {n_base * hours_per_row:.1f} h baseline, {n_up * hours_per_row:.1f} h upgraded") + if max_gap is not None: + ax.axvline( + max_gap / pd.Timedelta(minutes=1), + color="k", + linestyle=":", + label=f"pairing_max_gap {max_gap / pd.Timedelta(minutes=1):g} min", + ) + else: + text.append("no pairing filter configured") + ax.text(0.98, 0.95, "\n".join(text), transform=ax.transAxes, ha="right", va="top", fontsize=9) + ax.set_xlabel(f"gap to nearest used row of the other segment [min] (last bin: >= {cap_steps * step_min:g})") + ax.set_ylabel("used rows") + ax.set_yscale("log") + ax.set_title(f"{test}: gap from each used row to the opposite toggle state") + ax.grid(visible=True, alpha=0.3) + ax.legend(loc="upper right", bbox_to_anchor=(0.99, 0.83)) + fig.tight_layout() + _save(fig, path) + + def _save_per_bin_plot(path: Path, *, per_bin: pd.DataFrame, test: str, active_power_col: str) -> None: """Plot the per-power-bin uplift with each bin's used-record count underneath. diff --git a/benchmarking/harness/method.py b/benchmarking/harness/method.py index 1c17ace4..a70897a2 100644 --- a/benchmarking/harness/method.py +++ b/benchmarking/harness/method.py @@ -120,6 +120,10 @@ class MethodOutput: reference the method ruled out, so it stays visible without dragging that headline. :param screen_passes: optional per-pass frame from a method's reference-validity screen, so an analyst can see, and disagree with, which references it ruled out and why. + :param selection_accounting: optional per-(``stage``, ``segment``) frame of rows kept after each + of a method's row-selection stages (``n_kept``, ``kept_fraction``, ``stage_kept_fraction``), + so a consumer can see how much data each stage removed and whether it treated the segments + evenly. The harness never interprets it. """ p50_overall: float @@ -129,6 +133,7 @@ class MethodOutput: labeled_rows: pd.DataFrame | None = None reference_uplifts: pd.DataFrame | None = None screen_passes: pd.DataFrame | None = None + selection_accounting: pd.DataFrame | None = None @runtime_checkable diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index a13417b6..801856b5 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -24,6 +24,7 @@ _daily_segment_ratio, _expected_per_day, _infer_timebase, + pair_within, restrict_to_campaign, ) from benchmarking.harness.conditions import condition_bins @@ -801,13 +802,14 @@ def test_used_reproduces_the_methods_own_used_mask(self) -> None: wide = toggle_specialist._wide_column( # noqa: SLF001 mi_restricted.scada_df, turbine_col=_TURBINE_COL, value_col=_POWER_COL ) - expected = method._used_mask( # noqa: SLF001 + expected = method._selection( # noqa: SLF001 mi_restricted, wide=wide, test="T1", refs=[c for c in wide.columns if c != "T1"], timebase=pd.Timedelta(minutes=10), - ) + rows=resolve_toggle(mi_restricted.upgrade_timing, wide.index), + ).paired assert out.labeled_rows["used"].to_numpy().tolist() == expected.to_numpy().tolist() def test_a_downtime_row_is_labelled_unused(self) -> None: @@ -1107,3 +1109,150 @@ def test_rows_a_reference_may_not_contribute_are_dropped(self) -> None: ) untouched = self._estimate(scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) assert with_holes != pytest.approx(untouched) + + +# --- pairing filter (pairing_max_gap) ----------------------------------------------------------- + + +class TestPairWithin: + """``pair_within`` keeps a row only when the other segment has a row within the gap, inclusive.""" + + @staticmethod + def _one_each(gap_rows: int) -> tuple[pd.DatetimeIndex, np.ndarray, np.ndarray]: + index = _index(gap_rows + 1) + baseline = np.zeros(len(index), dtype=bool) + upgraded = np.zeros(len(index), dtype=bool) + baseline[0] = True + upgraded[-1] = True + return index, baseline, upgraded + + def test_a_gap_exactly_equal_to_the_limit_is_kept(self) -> None: + index, baseline, upgraded = self._one_each(gap_rows=2) + paired = pair_within(index, baseline=baseline, upgraded=upgraded, max_gap=pd.Timedelta(minutes=20)) + assert paired.baseline[0] + assert paired.upgraded[-1] + + def test_a_gap_one_row_beyond_the_limit_is_dropped(self) -> None: + index, baseline, upgraded = self._one_each(gap_rows=3) + paired = pair_within(index, baseline=baseline, upgraded=upgraded, max_gap=pd.Timedelta(minutes=20)) + assert not paired.baseline.any() + assert not paired.upgraded.any() + + def test_an_empty_other_segment_drops_everything(self) -> None: + index = _index(6) + baseline = np.ones(len(index), dtype=bool) + paired = pair_within( + index, baseline=baseline, upgraded=np.zeros(len(index), dtype=bool), max_gap=pd.Timedelta(hours=1) + ) + assert not paired.baseline.any() + + @pytest.mark.parametrize("gap_minutes", [10, 30, 60]) + def test_matches_v0_any_within_timedelta(self, gap_minutes: int) -> None: + """Parity with wind-up v0's ``_toggle_pairing_filter`` on a gappy toggle frame.""" + v0 = pytest.importorskip("wind_up_v0.main_analysis") + rng = np.random.default_rng(3) + full = _index(400) + index = full[rng.random(len(full)) > 0.3] # drop rows so gaps vary + state = rng.random(len(index)) > 0.5 + cols = {"ws": 1.0, "pw": 1.0, "wd": 1.0} + pre_df = pd.DataFrame(cols, index=index[~state]) + post_df = pd.DataFrame(cols, index=index[state]) + v0_pre, v0_post = v0._toggle_pairing_filter( # noqa: SLF001 - the v0 behaviour being matched + pre_df=pre_df, + post_df=post_df, + pairing_filter_method="any_within_timedelta", + pairing_filter_timedelta_seconds=gap_minutes * 60, + detrend_ws_col="ws", + test_pw_col="pw", + ref_wd_col="wd", + timebase_s=600, + ) + + paired = pair_within(index, baseline=~state, upgraded=state, max_gap=pd.Timedelta(minutes=gap_minutes)) + assert set(index[paired.baseline]) == set(v0_pre.index) + assert set(index[paired.upgraded]) == set(v0_post.index) + + +class TestPairingFilter: + """``pairing_max_gap`` on the method: validation, the rows it drops, and the selection accounting.""" + + @staticmethod + def _block_excluded_case() -> tuple[pd.DataFrame, ToggleSchedule, pd.DatetimeIndex]: + """A noisy toggle campaign whose upgraded test rows are all flagged over one 1-day block.""" + scada, schedule = _noisy_toggle_case() + idx = pd.DatetimeIndex(pd.unique(scada.index)) + treated = np.asarray(treated_mask(idx, schedule)) + block = (idx >= idx[500]) & (idx < idx[644]) + flagged = pd.Series(block & treated, index=idx) + out = scada.copy() + is_test = out[_TURBINE_COL] == "T1" + out[_EXCLUDE_COL] = False + out.loc[is_test, _EXCLUDE_COL] = flagged.reindex(out.index[is_test]).to_numpy() + return out, schedule, idx[block] + + @staticmethod + def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + return ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, **kwargs).estimate( + MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) + ) + + @pytest.mark.parametrize("gap", [pd.Timedelta(0), pd.Timedelta(minutes=5), pd.Timedelta(minutes=25)]) + def test_rejects_a_gap_that_is_not_a_positive_timebase_multiple(self, gap: pd.Timedelta) -> None: + scada, schedule = _noisy_toggle_case(n=200) + with pytest.raises(ValueError, match="pairing_max_gap"): + _estimate(scada, schedule, pairing_max_gap=gap) + + def test_baseline_rows_far_from_any_used_upgraded_row_are_dropped(self) -> None: + scada, schedule, block = self._block_excluded_case() + out = self._run(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) + assert out.labeled_rows is not None + rows = out.labeled_rows + inner = block[6:-6] # more than 20 min from the block's edges + assert not rows.loc[inner, "used"].astype(bool).any() + outside = rows.index < block[0] + assert rows.loc[outside, "used"].astype(bool).all() + + def test_without_pairing_the_block_keeps_its_baseline_rows(self) -> None: + scada, schedule, block = self._block_excluded_case() + out = self._run(scada, schedule) + assert out.labeled_rows is not None + rows = out.labeled_rows.loc[block] + assert rows.loc[rows["segment"] == "baseline", "used"].astype(bool).all() + + def test_accounting_shows_each_stage_per_segment(self) -> None: + scada, schedule, block = self._block_excluded_case() + out = self._run(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) + assert out.selection_accounting is not None + kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + n_block = len(block) // 2 + assert kept["exclude_row", "upgraded"] == kept["filters", "upgraded"] - n_block + assert kept["exclude_row", "baseline"] == kept["filters", "baseline"] + assert kept["pairing", "baseline"] < kept["exclude_row", "baseline"] + assert kept["pairing", "upgraded"] <= kept["exclude_row", "upgraded"] + + def test_accounting_is_reported_without_pairing(self) -> None: + scada, schedule = _noisy_toggle_case(n=200) + out = _estimate(scada, schedule) + assert out.selection_accounting is not None + kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + assert kept["pairing", "baseline"] == kept["exclude_row", "baseline"] + + def test_an_uneven_stage_warns(self, caplog: pytest.LogCaptureFixture) -> None: + scada, schedule, _ = self._block_excluded_case() + with caplog.at_level("WARNING", logger=toggle_specialist.__name__): + self._run(scada, schedule, segment_imbalance_warning=0.05) + assert any("exclude_row" in record.getMessage() for record in caplog.records) + + def test_an_even_selection_does_not_warn(self, caplog: pytest.LogCaptureFixture) -> None: + scada, schedule = _noisy_toggle_case(n=200) + with caplog.at_level("WARNING", logger=toggle_specialist.__name__): + _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) + assert not [r for r in caplog.records if r.name == toggle_specialist.__name__] + + +@pytest.mark.parametrize("gap", [None, pd.Timedelta(minutes=20)], ids=["no-pairing", "pairing"]) +def test_pairing_gap_plot_is_written(tmp_path: Path, gap: pd.Timedelta | None) -> None: + """The gap histogram is drawn with or without a pairing filter, so a user can judge whether one is needed.""" + scada, schedule = _noisy_toggle_case(n=200) + _estimate(scada, schedule, out_dir=tmp_path, save_plots=True, pairing_max_gap=gap) + assert len(list(tmp_path.rglob("*_pairing_gap.png"))) == 1 From 68d70980a42b26369d14d1860eb5d67e992ec155 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Thu, 24 Sep 2026 15:44:31 +0100 Subject: [PATCH 02/18] Read pairing gaps at nanosecond resolution whatever the index unit index.asi8 is in the index's own unit; real SCADA often arrives as datetime64[us], so a 10-minute gap read as 0.6 s and the pairing filter kept every row. Convert to ns first. Co-Authored-By: Claude Opus 5.5 --- benchmarking/baselines/toggle_specialist.py | 2 +- tests/benchmarking/baselines/test_toggle_specialist.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index d3402010..fa4de56d 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -110,7 +110,7 @@ def gap_to_other_segment( index: pd.DatetimeIndex, *, baseline: npt.NDArray[np.bool_], upgraded: npt.NDArray[np.bool_] ) -> npt.NDArray[np.float64]: """Seconds from each flagged row to the nearest flagged row of the other segment; NaN if unflagged.""" - times = index.asi8 + times = index.as_unit("ns").asi8 gap = np.full(len(index), np.nan) gap[baseline] = _nearest_gap_s(times[baseline], others=times[upgraded]) gap[upgraded] = _nearest_gap_s(times[upgraded], others=times[baseline]) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 801856b5..90b2f6d6 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -24,6 +24,7 @@ _daily_segment_ratio, _expected_per_day, _infer_timebase, + gap_to_other_segment, pair_within, restrict_to_campaign, ) @@ -1138,6 +1139,15 @@ def test_a_gap_one_row_beyond_the_limit_is_dropped(self) -> None: assert not paired.baseline.any() assert not paired.upgraded.any() + @pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) + def test_the_index_resolution_does_not_change_the_gap(self, unit: str) -> None: + """Real SCADA often arrives at microsecond resolution; a 10-min gap must still read as 10 min.""" + index, baseline, upgraded = self._one_each(gap_rows=1) + index = index.as_unit(unit) + assert gap_to_other_segment(index, baseline=baseline, upgraded=upgraded)[0] == pytest.approx(600.0) + paired = pair_within(index, baseline=baseline, upgraded=upgraded, max_gap=pd.Timedelta(minutes=5)) + assert not paired.baseline.any() + def test_an_empty_other_segment_drops_everything(self) -> None: index = _index(6) baseline = np.ones(len(index), dtype=bool) From bb7112997654c72c111e12a9d653836883accb2e Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 12:02:41 +0100 Subject: [PATCH 03/18] Add block reference modes and symmetric row exclusion to ToggleSpecialistMethod reference_mode selects what the test power is compared against: "sum" (the summed references, as before), "block_mean" (the mean test power of the row's own fixed wall-clock block, so no references are needed at all) or "block_scaled" (the summed references rescaled per block so each block's reference total equals its test total). Every used row of a block gets the same reference factor, so the on/off ratio cancels the block's wind level exactly; a block whose used rows lack either state is dropped and reported as a new "block" selection stage. The block length defaults to pairing_max_gap. The exclude_row column is now honoured on every turbine's rows, so a flagged reference row removes the timestamp the way a reference outage does. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/toggle_specialist.py | 225 ++++++++++++++---- .../baselines/test_toggle_specialist.py | 115 ++++++++- 2 files changed, 286 insertions(+), 54 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index fa4de56d..312aa6c7 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -17,6 +17,15 @@ other segment's rows from those conditions. Rows kept per segment after each selection stage are reported as ``MethodOutput.selection_accounting`` and written to a selection CSV. +**Reference modes** (``reference_mode``) decide what the test power is compared against. ``"sum"`` +is the summed power of the references. The two *block* modes tile the campaign into fixed +wall-clock blocks of ``reference_block`` (one toggle cycle by default) and give every used row of a +block the same reference value, so the on/off ratio cancels the block's wind level exactly: +``"block_mean"`` uses the block's mean test power and needs **no references at all**; +``"block_scaled"`` keeps the summed references but rescales them per block so each block's reference +total equals its test total, leaving the references only the within-block tracking. A block whose +used rows lack either state is dropped (the ``block`` selection stage). + Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the uplift, from the uplift's own frozen row selection and bin assignment, and only when the uplift is @@ -74,6 +83,8 @@ # treatment cannot move a row between bins. Binning by the test turbine's ws/TI would condition on # post-treatment signals, which this method exists not to do (see the module docstring). _SUPPORTED_CONDITIONS: tuple[str, ...] = ("power",) +# ``reference_mode`` values; see the module docstring. +REFERENCE_MODES: tuple[str, ...] = ("sum", "block_mean", "block_scaled") logger = logging.getLogger(__name__) @@ -84,6 +95,7 @@ class _Selection(NamedTuple): filtered: pd.Series not_excluded: pd.Series paired: pd.Series + blocked: pd.Series class PairedRows(NamedTuple): @@ -135,6 +147,29 @@ def _infer_timebase(index: pd.DatetimeIndex) -> pd.Timedelta: return pd.Timedelta(np.median(np.diff(unique.to_numpy()))) +def block_ids(index: pd.DatetimeIndex, *, start: pd.Timestamp, block: pd.Timedelta) -> npt.NDArray[np.int64]: + """Index of the fixed wall-clock block of length ``block``, tiled forward from ``start``, holding each row.""" + return np.floor(np.asarray((index - start) / block, dtype=float)).astype(np.int64) + + +def blocks_with_both_states( + ids: npt.NDArray[np.int64], *, baseline: npt.NDArray[np.bool_], upgraded: npt.NDArray[np.bool_] +) -> npt.NDArray[np.bool_]: + """Per row: whether its block holds at least one ``baseline`` and one ``upgraded`` row.""" + both = set(np.unique(ids[baseline])) & set(np.unique(ids[upgraded])) + return np.isin(ids, list(both)) + + +def _block_sum(ids: npt.NDArray[np.int64], values: npt.NDArray[np.float64], mask: npt.NDArray[np.bool_]) -> pd.Series: + """Sum of ``values`` over the ``mask`` rows of each block, indexed by block id.""" + return pd.Series(values[mask]).groupby(ids[mask]).sum() + + +def _per_row(ids: npt.NDArray[np.int64], per_block: pd.Series) -> npt.NDArray[np.float64]: + """Broadcast a per-block value back onto rows; NaN for a row whose block has no value.""" + return per_block.reindex(ids).to_numpy(dtype=float) + + def _wide_column(scada_df: pd.DataFrame, *, turbine_col: str, value_col: str) -> pd.DataFrame: """Pivot long SCADA to a timestamp x turbine table of ``value_col`` (NaN where missing).""" tmp = scada_df[[turbine_col, value_col]].copy() @@ -189,6 +224,10 @@ class ToggleSpecialistMethod: a positive whole multiple of the timebase; ``None`` disables pairing. :param segment_imbalance_warning: warn when the two segments' kept fractions at any selection stage differ by more than this. + :param reference_mode: one of :data:`REFERENCE_MODES` (see the module docstring). ``"block_mean"`` + uses no references: they do not enter the row filters either, and none need exist. + :param reference_block: block length for the block modes, a positive whole multiple of the + timebase; defaults to ``pairing_max_gap`` (one cycle). Ignored in ``"sum"`` mode. """ columns: ColumnSchema @@ -203,10 +242,15 @@ class ToggleSpecialistMethod: bootstrap_seed: int = 0 pairing_max_gap: pd.Timedelta | None = None segment_imbalance_warning: float = 0.1 + reference_mode: str = "sum" + reference_block: pd.Timedelta | None = None def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" self.columns.require_roles(("active_power", "availability")) + if self.reference_mode not in REFERENCE_MODES: + msg = f"{self.name}: reference_mode {self.reference_mode!r} is not one of {REFERENCE_MODES}" + raise ValueError(msg) validate_conditions(self.conditions, supported=_SUPPORTED_CONDITIONS, method_name=self.name) if "power" in self.conditions and self.rated_power_kw is None: msg = ( @@ -228,11 +272,14 @@ def estimate(self, mi: MethodInput) -> MethodOutput: mi = restrict_to_campaign(mi) wide = _wide_column(mi.scada_df, turbine_col=mi.turbine_col, value_col=self.columns.active_power) test = mi.test_wtg - refs = mi.context.references_among(wide.columns) - if not refs: + # block_mean compares the test turbine against its own block, so references play no part, + # not even in the row filters: a reference outage must not cost a zero-reference run rows. + refs = [] if self.reference_mode == "block_mean" else mi.context.references_among(wide.columns) + if not refs and self.reference_mode != "block_mean": msg = ( f"no reference turbines available for test_wtg {test!r}: scada_df contains only " - f"{list(wide.columns)}. The toggle specialist method needs at least one reference turbine." + f"{list(wide.columns)}. The toggle specialist method needs at least one reference turbine " + f"in reference_mode {self.reference_mode!r} (only 'block_mean' runs without references)." ) raise ValueError(msg) # Narrow to the campaign's turbines and blank the cells it says may not contribute, so the @@ -249,13 +296,20 @@ def estimate(self, mi: MethodInput) -> MethodOutput: timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) self._check_pairing_gap(timebase) + block = self._reference_block(timebase) rows = resolve_toggle(mi.upgrade_timing, wide.index) baseline = rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) - ref_total = wide[refs].sum(axis=1).to_numpy(dtype=float) - selection = self._selection(mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows) - used = selection.paired.to_numpy() + campaign = baseline | rows.upgraded + ids = ( + block_ids(wide.index, start=wide.index[campaign].min(), block=block) + if block is not None and campaign.any() + else None + ) + selection = self._selection(mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids) + used = selection.blocked.to_numpy() accounting = self._selection_accounting(selection, rows=rows) + ref_total = self._reference_total(wide=wide, refs=refs, test_pw=test_pw, used=used, ids=ids) rho_base = _rho(test_pw, ref_total, used & baseline) rho_up = _rho(test_pw, ref_total, used & rows.upgraded) @@ -307,7 +361,7 @@ def estimate(self, mi: MethodInput) -> MethodOutput: wide=wide, used=used, toggle_rows=rows, - refs=refs, + ref_total=ref_total, timebase=timebase, active_power_col=self.columns.active_power, ) @@ -317,11 +371,13 @@ def estimate(self, mi: MethodInput) -> MethodOutput: wide=wide, stats=stats, used=used, + ref_total=ref_total, rho_base=rho_base, rho_up=rho_up, uplift=uplift, sigma_overall=sigma_overall, n_refs=len(refs), + refs=refs, timebase=timebase, per_bin=per_bin, diagnostics=diagnostics, @@ -478,6 +534,53 @@ def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: ) raise ValueError(msg) + def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta | None: + """Return the block length the block modes tile the campaign with; ``None`` in ``sum`` mode.""" + if self.reference_mode == "sum": + return None + block = self.reference_block if self.reference_block is not None else self.pairing_max_gap + if block is None: + msg = ( + f"{self.name}: reference_mode {self.reference_mode!r} needs a block length: set reference_block " + f"or pairing_max_gap (one toggle cycle is the natural choice)." + ) + raise ValueError(msg) + ratio = block / timebase + if block <= pd.Timedelta(0) or ratio != round(ratio): + msg = ( + f"{self.name}: reference_block {block} must be a positive whole multiple of the timebase " + f"{timebase}; a block that does not tile the timebase grid cannot hold whole records." + ) + raise ValueError(msg) + return block + + def _reference_total( + self, + *, + wide: pd.DataFrame, + refs: list[str], + test_pw: npt.NDArray[np.float64], + used: npt.NDArray[np.bool_], + ids: npt.NDArray[np.int64] | None, + ) -> npt.NDArray[np.float64]: + """Return the per-row reference power for ``reference_mode``; NaN on rows no surviving block covers. + + Every used row of a block gets the same block-level factor, so within a block the on/off + ratio cancels the block's wind level whatever the on/off count mix (the block mean carries + the uplift at ``n_on / n`` strength in both denominators, a second-order effect). + """ + ref_sum = wide[refs].sum(axis=1).to_numpy(dtype=float) if refs else np.zeros(len(wide)) + if self.reference_mode == "sum": + return ref_sum + if ids is None or not used.any(): + return np.full(len(wide), np.nan) + block_test = _block_sum(ids, test_pw, used) + if self.reference_mode == "block_mean": + n_used = pd.Series(used.astype(float)).groupby(ids).sum() + return _per_row(ids, block_test / n_used.reindex(block_test.index)) + scale = block_test / _block_sum(ids, ref_sum, used) + return ref_sum * _per_row(ids, scale) + def _selection( self, mi: MethodInput, @@ -487,22 +590,35 @@ def _selection( refs: list[str], timebase: pd.Timedelta, rows: ToggleRowSets, + ids: npt.NDArray[np.int64] | None = None, ) -> _Selection: - """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``.""" + """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``. + + ``ids`` are the reference-block ids (``None`` outside the block modes, or with no campaign + rows to tile); the ``block`` stage drops the rows of a block that lacks either state. + """ filtered = self._filtered_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase) - not_excluded = filtered & ~self._test_excluded(mi, test=test, index=wide.index) - if self.pairing_max_gap is None: - return _Selection(filtered=filtered, not_excluded=not_excluded, paired=not_excluded) + not_excluded = filtered & ~self._excluded(mi, turbines=[test, *refs], index=wide.index) + in_segment = rows.campaign_baseline | rows.upgraded kept = not_excluded.to_numpy() - paired = pair_within( - wide.index, - baseline=kept & rows.campaign_baseline, - upgraded=kept & rows.upgraded, - max_gap=self.pairing_max_gap, + if self.pairing_max_gap is not None: + paired = pair_within( + wide.index, + baseline=kept & rows.campaign_baseline, + upgraded=kept & rows.upgraded, + max_gap=self.pairing_max_gap, + ) + kept = kept & (~in_segment | paired.baseline | paired.upgraded) + paired_series = pd.Series(kept, index=wide.index) + if ids is not None: + both = blocks_with_both_states(ids, baseline=kept & rows.campaign_baseline, upgraded=kept & rows.upgraded) + kept = kept & (~in_segment | both) + return _Selection( + filtered=filtered, + not_excluded=not_excluded, + paired=paired_series, + blocked=pd.Series(kept, index=wide.index), ) - in_segment = rows.campaign_baseline | rows.upgraded - paired_mask = kept & (~in_segment | paired.baseline | paired.upgraded) - return _Selection(filtered=filtered, not_excluded=not_excluded, paired=pd.Series(paired_mask, index=wide.index)) def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) -> pd.DataFrame: """Rows kept per segment after each stage, warning when a stage treats the segments unevenly. @@ -515,6 +631,7 @@ def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) - ("filters", selection.filtered), ("exclude_row", selection.not_excluded), ("pairing", selection.paired), + ("block", selection.blocked), ) records = [] for segment, in_segment in ((_BASELINE, rows.campaign_baseline), (_UPGRADED, rows.upgraded)): @@ -582,27 +699,30 @@ def _filtered_mask( ) return complete & all_available & test_keep - def _test_excluded(self, mi: MethodInput, *, test: str, index: pd.DatetimeIndex) -> pd.Series: - """Boolean mask on *index*: the test turbine's caller-flagged rows to drop (empty when unused). + def _excluded(self, mi: MethodInput, *, turbines: list[str], index: pd.DatetimeIndex) -> pd.Series: + """Boolean mask on *index*: timestamps at which any of ``turbines`` carries a caller-set exclusion flag. - Reads the ``columns.exclude_row`` column of the test turbine's rows; references are never - excluded here (their special modes still carry information for the ratio). Absent column or - unset role -> nothing excluded. Reindex fills missing timestamps with ``False`` so an expanded - index never becomes an exclusion. NaN raises rather than coercing: ``astype(bool)`` reads a - missing flag as ``True`` and drops the row, the opposite of the safe default. + Reads the ``columns.exclude_row`` column of each listed turbine's rows (the test turbine and + the references the mode uses), so a flagged reference row removes the timestamp the way a + reference outage does. Absent column or unset role -> nothing excluded. Reindex fills missing + timestamps with ``False`` so an expanded index never becomes an exclusion. NaN raises rather + than coercing: ``astype(bool)`` reads a missing flag as ``True`` and drops the row, the + opposite of the safe default. """ col = self.columns.exclude_row + excluded = pd.Series(data=False, index=index, dtype=bool) if not col or col not in mi.scada_df.columns: - return pd.Series(data=False, index=index, dtype=bool) - test_rows = mi.scada_df[mi.scada_df[mi.turbine_col] == test] - flags = test_rows[col] - if flags.isna().any(): - msg = ( - f"exclude_row column {col!r} has {int(flags.isna().sum())} NaN value(s) for turbine {test!r}; " - f"it must be boolean with no missing values (fill unknown rows with False explicitly)" - ) - raise ValueError(msg) - return flags.astype(bool).reindex(index, fill_value=False) + return excluded + for turbine in turbines: + flags = mi.scada_df.loc[mi.scada_df[mi.turbine_col] == turbine, col] + if flags.isna().any(): + msg = ( + f"exclude_row column {col!r} has {int(flags.isna().sum())} NaN value(s) for turbine " + f"{turbine!r}; it must be boolean with no missing values (fill unknown rows with False explicitly)" + ) + raise ValueError(msg) + excluded |= flags.astype(bool).reindex(index, fill_value=False) + return excluded def _write_outputs( self, @@ -611,11 +731,13 @@ def _write_outputs( wide: pd.DataFrame, stats: pd.DataFrame, used: np.ndarray, + ref_total: npt.NDArray[np.float64], rho_base: float, rho_up: float, uplift: float, sigma_overall: float, n_refs: int, + refs: list[str], timebase: pd.Timedelta, per_bin: pd.DataFrame | None = None, diagnostics: pd.DataFrame | None = None, @@ -645,6 +767,7 @@ def _write_outputs( "mode": "toggle", "n_turbines": wide.shape[1], "n_refs": n_refs, + "reference_mode": self.reference_mode, "ratio_baseline": rho_base, "ratio_upgraded": rho_up, "uplift_frc": uplift, @@ -673,6 +796,8 @@ def _write_outputs( mi=mi, test=mi.test_wtg, used=used, + ref_total=ref_total, + reference_mode=self.reference_mode, timebase=timebase, active_power_col=self.columns.active_power, ) @@ -693,16 +818,24 @@ def _write_outputs( max_gap=self.pairing_max_gap, test=mi.test_wtg, ) - self._write_shared_diagnostics(mi, run_dir=run_dir, wide=wide, used=used, timebase=timebase) + self._write_shared_diagnostics( + mi, run_dir=run_dir, wide=wide, used=used, timebase=timebase, turbines=[mi.test_wtg, *refs] + ) def _write_shared_diagnostics( - self, mi: MethodInput, *, run_dir: Path, wide: pd.DataFrame, used: np.ndarray, timebase: pd.Timedelta + self, + mi: MethodInput, + *, + run_dir: Path, + wide: pd.DataFrame, + used: np.ndarray, + timebase: pd.Timedelta, + turbines: list[str], ) -> None: """Emit the shared cross-method diagnostics (coverage/curves/histograms) and the run config.""" # ``wide`` (a pivot) drops all-NaN timestamps, so align the masks to the full unique index # the DiagnosticContext uses (timestamps absent from ``wide`` are simply not used). index = pd.DatetimeIndex(pd.unique(mi.scada_df.index)).sort_values() - test = mi.test_wtg used = pd.Series(used, index=wide.index).reindex(index, fill_value=False).to_numpy() treated = resolve_toggle(mi.upgrade_timing, index).upgraded.astype(bool) ctx = DiagnosticContext( @@ -717,13 +850,15 @@ def _write_shared_diagnostics( mode="toggle", era5_df=None, # the exclusion alone, so the plots show what the flag removed that downtime did not - excluded_ts=self._test_excluded(mi, test=test, index=index).to_numpy(), + excluded_ts=self._excluded(mi, turbines=turbines, index=index).to_numpy(), ) write_common_diagnostics(ctx) params = { "active_power_col": self.columns.active_power, "availability_col": self.columns.availability, "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), + "reference_mode": self.reference_mode, + "reference_block": None if self.reference_block is None else str(self.reference_block), } write_run_config(ctx, method_name=self.name, method_params=params) @@ -823,14 +958,13 @@ def _segment_stats( wide: pd.DataFrame, used: npt.NDArray[np.bool_], toggle_rows: ToggleRowSets, - refs: list[str], + ref_total: npt.NDArray[np.float64], timebase: pd.Timedelta, active_power_col: str, ) -> pd.DataFrame: """Build the per-segment (all/baseline/upgraded) diagnostics table.""" test = mi.test_wtg test_pw = wide[test].to_numpy(dtype=float) - ref_total = wide[refs].sum(axis=1).to_numpy(dtype=float) n_turbines = wide.shape[1] timebase_hours = timebase / pd.Timedelta(hours=1) @@ -935,6 +1069,8 @@ def _save_plots( mi: MethodInput, test: str, used: np.ndarray, + ref_total: npt.NDArray[np.float64], + reference_mode: str, timebase: pd.Timedelta, active_power_col: str, ) -> None: @@ -945,11 +1081,10 @@ def _save_plots( The baseline is the strict campaign off-blocks the estimate used, so the plots never disagree with the headline. """ - refs = [c for c in wide.columns if c != test] toggle_rows = resolve_toggle(mi.upgrade_timing, wide.index) baseline_mask = toggle_rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) - ref_total = wide[refs].sum(axis=1).to_numpy(dtype=float) + ref_label = f"sum of reference {active_power_col}" if reference_mode == "sum" else f"{reference_mode} reference" upgrade_start = toggle_upgrade_start(mi.upgrade_timing, wide.index) segments = ( ("baseline", used & baseline_mask, "C0"), @@ -964,7 +1099,7 @@ def _save_plots( if np.isfinite(rho) and seg.any(): x_max = float(np.nanmax(ref_total[seg])) ax.plot([0, x_max], [0, rho * x_max], color=color, linewidth=1.5) - ax.set_xlabel(f"sum of reference {active_power_col} [kW]") + ax.set_xlabel(f"{ref_label} [kW]") ax.set_ylabel(f"{active_power_col} @ {test} [kW]") ax.set_title(f"{test}: test vs reference-total power") ax.grid(visible=True, alpha=0.3) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 90b2f6d6..d705af53 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -981,19 +981,19 @@ def test_excluding_corrupted_rows_restores_the_known_uplift(self) -> None: assert filtered.p50_overall == pytest.approx(0.05, abs=1e-9) assert abs(naive.p50_overall - 0.05) > abs(filtered.p50_overall - 0.05) - def test_flagged_reference_rows_are_kept(self) -> None: - """References are never excluded here: their special modes still carry ratio information.""" + def test_flagged_reference_rows_drop_the_timestamp(self) -> None: + """A flagged reference row (curtailed, iced, parked) removes the timestamp like a reference outage.""" scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="R1", every=3) - baseline = ToggleSpecialistMethod(columns=_COLUMNS).estimate( - MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) - with_role = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - assert with_role.p50_overall == pytest.approx(baseline.p50_overall) - assert with_role.labeled_rows is not None - assert with_role.labeled_rows["used"].to_numpy().any() + assert out.labeled_rows is not None + is_ref = (flagged[_TURBINE_COL] == "R1").to_numpy() + flagged_ts = flagged.index[is_ref & flagged[_EXCLUDE_COL].to_numpy(dtype=bool)] + used = out.labeled_rows["used"].astype(bool) + assert not used.loc[flagged_ts].any() + assert used.drop(flagged_ts).any() def test_nan_in_the_exclude_column_is_rejected(self) -> None: """NaN is not "excluded": the schema contract says the column is never NaN, so say so loudly.""" @@ -1266,3 +1266,100 @@ def test_pairing_gap_plot_is_written(tmp_path: Path, gap: pd.Timedelta | None) - scada, schedule = _noisy_toggle_case(n=200) _estimate(scada, schedule, out_dir=tmp_path, save_plots=True, pairing_max_gap=gap) assert len(list(tmp_path.rglob("*_pairing_gap.png"))) == 1 + + +# --- reference modes: block_mean (zero references) and block_scaled ------------------------------ + +_BLOCK = pd.Timedelta(minutes=40) # one full 20-min on / 20-min off cycle = 4 rows of 10 min + + +def _block_case( + n_blocks: int = 120, + *, + uplift: float = 0.04, + ramp: float = 0.0, + ref_scale: np.ndarray | None = None, + seed: int = 0, +) -> tuple[pd.DataFrame, ToggleSchedule]: + """A toggle campaign whose power level jumps wildly between 40-min blocks (two on/off cycles each). + + Test power is ``level_b * shape_i``, times ``1 + uplift`` when treated, where ``shape`` is a + within-block ramp of slope ``ramp`` per row (flat by default: a ramp in phase with the toggle + is exactly the drift ``block_mean`` cannot cancel). With ``ref_scale`` (one factor per block) a + reference ``R1 = ref_scale_b * level_b * shape_i`` is added: it tracks the test turbine inside + every block but its ratio to the test changes from block to block. + """ + rng = np.random.default_rng(seed) + idx = _index(4 * n_blocks) + schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=idx[0]) + treated = np.asarray(treated_mask(idx, schedule)) + level = np.repeat(rng.uniform(200.0, 2000.0, n_blocks), 4) + shape = np.tile(1.0 + ramp * np.arange(4), n_blocks) + base = level * shape + test = np.where(treated, base * (1.0 + uplift), base) + turbines = {"T1": test} + if ref_scale is not None: + turbines["R1"] = np.repeat(ref_scale, 4) * base + return _scada(turbines, idx), schedule + + +class TestReferenceModes: + def test_block_mean_recovers_the_uplift_with_no_references(self) -> None: + scada, schedule = _block_case(uplift=0.04) + out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + assert np.isfinite(out.sigma_overall) + + def test_block_mean_ignores_a_reference_outage(self) -> None: + """Zero-reference mode must not pay for references it does not use.""" + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() + scada.loc[is_ref, _POWER_COL] = np.nan + out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + assert out.labeled_rows is not None + assert out.labeled_rows["used"].astype(bool).all() + + def test_block_scaled_recovers_the_uplift_when_the_reference_ratio_drifts_between_blocks(self) -> None: + rng = np.random.default_rng(1) + scada, schedule = _block_case(uplift=0.04, ramp=0.1, ref_scale=rng.uniform(0.3, 3.0, 120)) + out = _estimate(scada, schedule, reference_mode="block_scaled", reference_block=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + + def test_block_length_defaults_to_the_pairing_gap(self) -> None: + scada, schedule = _block_case(uplift=0.04) + out = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + + def test_a_block_with_one_state_is_dropped_and_accounted(self) -> None: + scada, schedule = _block_case(uplift=0.04) + idx = pd.DatetimeIndex(pd.unique(scada.index)) + block_rows = idx[40:44] # the 11th block: two on/off cycles + off_rows = block_rows[~np.asarray(treated_mask(block_rows, schedule))] + scada.loc[scada.index.isin(off_rows), _POWER_COL] = np.nan + out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + assert out.labeled_rows is not None + # NaN (a timestamp absent from the pivot) must read as unused, so compare rather than cast + used = out.labeled_rows["used"] == True # noqa: E712 + assert not used.loc[block_rows].any() + assert used.sum() == len(idx) - 4 + kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + assert kept["block", "upgraded"] == kept["pairing", "upgraded"] - 2 + assert kept["block", "baseline"] == kept["pairing", "baseline"] + + def test_sum_mode_reports_a_block_stage_that_keeps_everything(self) -> None: + scada, schedule = _noisy_toggle_case(n=200) + out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) + kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + stages = ["segment", "filters", "exclude_row", "pairing", "block"] + assert list(out.selection_accounting["stage"].unique()) == stages + assert kept["block", "baseline"] == kept["pairing", "baseline"] + + def test_unknown_mode_raises(self) -> None: + with pytest.raises(ValueError, match="reference_mode"): + ToggleSpecialistMethod(columns=_COLUMNS, reference_mode="bogus") + + def test_a_block_mode_without_a_block_length_raises(self) -> None: + scada, schedule = _block_case(n_blocks=10, ref_scale=np.ones(10)) + with pytest.raises(ValueError, match="reference_block"): + _estimate(scada, schedule, reference_mode="block_scaled") From dbaa8bec87344b9502d67d5bfefe0f4142395ec0 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 13:32:32 +0100 Subject: [PATCH 04/18] Add a combined reference mode that blends the sum and block_mean estimates reference_mode="combined" runs the sum and block_mean estimates, each on its own row selection, and blends them by inverse variance, headline and per bin. The bootstrap now exposes each cell's resampled uplifts; two bootstraps with the same campaign span, timebase, block length, resample count and seed draw the same blocks, so combine_estimates can take the components' correlation from their draw-paired resamples and report an honest sigma for the blend. A window with no references at all falls back to block_mean with a warning. The uncertainty diagnostics gain the reported sigma and a component column; the selection CSV lists every component's selection. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 76 +++- benchmarking/baselines/toggle_specialist.py | 343 ++++++++++++------ .../baselines/test_block_bootstrap.py | 69 +++- .../baselines/test_toggle_specialist.py | 44 +++ 4 files changed, 427 insertions(+), 105 deletions(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 3a2add4e..2b3bc308 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -15,7 +15,7 @@ from __future__ import annotations import math -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING import numpy as np @@ -74,10 +74,83 @@ class BootstrapResult: :param n_blocks: blocks drawn per resample (``ceil(campaign / block)``) :param cells: uncertainty per cell name, keyed as the caller keyed ``cell_membership`` + :param resamples: per cell, the resampled uplifts themselves (NaN where a resample was + degenerate), in draw order; empty when the bootstrap did not run. Two bootstraps run with the + same campaign span, timebase, block length, resample count and seed draw the same blocks, so + their resamples pair up draw by draw (see :func:`combine_estimates`). """ n_blocks: int cells: dict[str, CellUncertainty] + resamples: dict[str, npt.NDArray[np.float64]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CombinedEstimate: + """An inverse-variance blend of two estimates of the same uplift. + + :param estimate: ``weight_a * a + (1 - weight_a) * b`` + :param sigma: the blend's 1-sigma, including the two components' correlation + :param weight_a: the weight on the first component, in ``[0, 1]``; NaN when neither was usable + :param correlation: the correlation of the two components' paired resamples that ``sigma`` used + """ + + estimate: float + sigma: float + weight_a: float + correlation: float + + +# Correlation assumed between two components when their resamples cannot be paired: midway between +# independent and identical, so the blend neither claims the full inverse-variance gain nor none. +_DEFAULT_COMPONENT_CORRELATION = 0.5 + + +def combine_estimates( + a: tuple[float, float, npt.NDArray[np.float64]], b: tuple[float, float, npt.NDArray[np.float64]] +) -> CombinedEstimate: + """Blend two ``(estimate, sigma, resamples)`` of the same quantity by inverse variance. + + A component is usable when both its estimate and its sigma are finite. With two usable + components the weights are ``1 / sigma**2`` (a zero sigma takes all the weight) and the blend's + variance is ``(wa**2 sa**2 + wb**2 sb**2 + 2 wa wb r sa sb) / (wa + wb)**2`` with ``r`` the + Pearson correlation of the draw-paired finite resamples, or + :data:`_DEFAULT_COMPONENT_CORRELATION` when fewer than two pairs exist. One usable component is + returned as is; none gives NaN throughout. + """ + est_a, sig_a, res_a = a + est_b, sig_b, res_b = b + usable_a = math.isfinite(est_a) and math.isfinite(sig_a) + usable_b = math.isfinite(est_b) and math.isfinite(sig_b) + nan = float("nan") + if usable_a and not usable_b: + return CombinedEstimate(estimate=est_a, sigma=sig_a, weight_a=1.0, correlation=nan) + if usable_b and not usable_a: + return CombinedEstimate(estimate=est_b, sigma=sig_b, weight_a=0.0, correlation=nan) + if not (usable_a and usable_b): + return CombinedEstimate(estimate=nan, sigma=nan, weight_a=nan, correlation=nan) + + if sig_a == 0.0 or sig_b == 0.0: + weight_a = 0.5 if sig_a == sig_b else float(sig_a == 0.0) + else: + wa, wb = 1.0 / sig_a**2, 1.0 / sig_b**2 + weight_a = wa / (wa + wb) + weight_b = 1.0 - weight_a + + n_pairs = min(len(res_a), len(res_b)) + paired = np.isfinite(res_a[:n_pairs]) & np.isfinite(res_b[:n_pairs]) + if paired.sum() >= _MIN_RESAMPLES_FOR_SPREAD: + r = float(np.corrcoef(res_a[:n_pairs][paired], res_b[:n_pairs][paired])[0, 1]) + correlation = r if math.isfinite(r) else _DEFAULT_COMPONENT_CORRELATION + else: + correlation = _DEFAULT_COMPONENT_CORRELATION + variance = weight_a**2 * sig_a**2 + weight_b**2 * sig_b**2 + 2.0 * weight_a * weight_b * correlation * sig_a * sig_b + return CombinedEstimate( + estimate=weight_a * est_a + weight_b * est_b, + sigma=math.sqrt(max(variance, 0.0)), + weight_a=weight_a, + correlation=correlation, + ) def _nan_cells(names: list[str]) -> dict[str, CellUncertainty]: @@ -227,6 +300,7 @@ def bootstrap_ratio_uplift( return BootstrapResult( n_blocks=n_blocks, cells=_summarise(uplift, names=names, fallback=fallback, boot_weight=boot_weight), + resamples={name: uplift[:, i] for i, name in enumerate(names)}, ) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 312aa6c7..5241fe4e 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -24,7 +24,12 @@ ``"block_mean"`` uses the block's mean test power and needs **no references at all**; ``"block_scaled"`` keeps the summed references but rescales them per block so each block's reference total equals its test total, leaving the references only the within-block tracking. A block whose -used rows lack either state is dropped (the ``block`` selection stage). +used rows lack either state is dropped (the ``block`` selection stage). ``"combined"`` runs the +``sum`` and ``block_mean`` estimates, each on its own row selection, and blends them by inverse +variance, headline and per bin; the two bootstraps share their block draws, so the blend's sigma +includes the components' correlation (:func:`benchmarking.baselines.block_bootstrap.combine_estimates`). +Their errors come from different places (reference mismatch versus within-cycle drift), which is +what the blend exploits. Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the @@ -53,7 +58,12 @@ import pandas as pd from matplotlib.ticker import PercentFormatter -from benchmarking.baselines.block_bootstrap import BootstrapResult, bootstrap_ratio_uplift +from benchmarking.baselines.block_bootstrap import ( + BootstrapResult, + CombinedEstimate, + bootstrap_ratio_uplift, + combine_estimates, +) from benchmarking.baselines.filtering import NormalOperationFilter from benchmarking.diagnostics import DiagnosticContext, stages, write_common_diagnostics, write_run_config from benchmarking.harness.conditions import condition_bins, energy_ratio_by_bin, validate_conditions @@ -84,7 +94,10 @@ # post-treatment signals, which this method exists not to do (see the module docstring). _SUPPORTED_CONDITIONS: tuple[str, ...] = ("power",) # ``reference_mode`` values; see the module docstring. -REFERENCE_MODES: tuple[str, ...] = ("sum", "block_mean", "block_scaled") +REFERENCE_MODES: tuple[str, ...] = ("sum", "block_mean", "block_scaled", "combined") +# The two estimates ``"combined"`` blends, in blend order (the first is the one the row-selection +# accounting and the reference-side diagnostics report). +_COMBINED_COMPONENTS: tuple[str, str] = ("sum", "block_mean") logger = logging.getLogger(__name__) @@ -98,6 +111,26 @@ class _Selection(NamedTuple): blocked: pd.Series +@dataclass(frozen=True) +class _Estimate: + """One reference mode's complete estimate: what the outputs, the labels and the blend read.""" + + mode: str + refs: list[str] + selection: _Selection + used: npt.NDArray[np.bool_] + ref_total: npt.NDArray[np.float64] + label: npt.NDArray[np.float64] + rho_base: float + rho_up: float + uplift: float + sigma_overall: float + per_bin: pd.DataFrame | None + boot: BootstrapResult | None + accounting: pd.DataFrame + diagnostics: pd.DataFrame + + class PairedRows(NamedTuple): """The rows of each segment that have a row of the other segment within the pairing gap.""" @@ -268,48 +301,105 @@ def estimate(self, mi: MethodInput) -> MethodOutput: f"or toggle_df; this method has no prepost baseline to compare against." ) raise ValueError(msg) + if self.columns.availability not in mi.scada_df.columns: + msg = ( + f"the availability column {self.columns.availability!r} (columns.availability) is not in " + f"scada_df; the downtime filter is required for the toggle specialist method and cannot be skipped." + ) + raise ValueError(msg) mi = restrict_to_campaign(mi) - wide = _wide_column(mi.scada_df, turbine_col=mi.turbine_col, value_col=self.columns.active_power) + wide_all = _wide_column(mi.scada_df, turbine_col=mi.turbine_col, value_col=self.columns.active_power) test = mi.test_wtg + timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) + self._check_pairing_gap(timebase) + block = self._reference_block(timebase) + rows = resolve_toggle(mi.upgrade_timing, wide_all.index) + + modes: tuple[str, ...] = (self.reference_mode,) + if self.reference_mode == "combined": + modes = _COMBINED_COMPONENTS + if not mi.context.references_among(wide_all.columns): + # Nothing for the reference-based component to work with: the blend is its other + # half alone, which needs no references, rather than a failed estimate. + logger.warning( + "%s: no reference turbines available for test_wtg %r; 'combined' falls back to 'block_mean'", + self.name, + test, + ) + modes = ("block_mean",) + components = { + mode: self._component( + mi, mode=mode, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + ) + for mode in modes + } + est = ( + _combine(components[_COMBINED_COMPONENTS[0]], components[_COMBINED_COMPONENTS[1]]) + if modes == _COMBINED_COMPONENTS + else components[modes[0]] + ) + + ref_set = set(est.refs) + wide = wide_all[[c for c in wide_all.columns if c == test or c in ref_set]] + stats = _segment_stats( + mi, + wide=wide, + used=est.used, + toggle_rows=rows, + ref_total=est.ref_total, + timebase=timebase, + active_power_col=self.columns.active_power, + ) + self._write_outputs(mi, wide=wide, stats=stats, est=est, components=components, timebase=timebase, rows=rows) + return MethodOutput( + p50_overall=float(est.uplift), + p50_by_condition=est.per_bin, + sigma_overall=est.sigma_overall, + uncertainty_diagnostics=est.diagnostics, + labeled_rows=self._labeled_rows(mi, wide=wide, test=test, used=est.used, rows=rows, label=est.label), + selection_accounting=est.accounting, + ) + + def _component( + self, + mi: MethodInput, + *, + mode: str, + wide_all: pd.DataFrame, + test: str, + timebase: pd.Timedelta, + block: pd.Timedelta | None, + rows: ToggleRowSets, + ) -> _Estimate: + """Run one reference mode end to end: row selection, reference, uplift, bins and bootstrap.""" # block_mean compares the test turbine against its own block, so references play no part, # not even in the row filters: a reference outage must not cost a zero-reference run rows. - refs = [] if self.reference_mode == "block_mean" else mi.context.references_among(wide.columns) - if not refs and self.reference_mode != "block_mean": + refs = [] if mode == "block_mean" else mi.context.references_among(wide_all.columns) + if not refs and mode != "block_mean": msg = ( f"no reference turbines available for test_wtg {test!r}: scada_df contains only " - f"{list(wide.columns)}. The toggle specialist method needs at least one reference turbine " - f"in reference_mode {self.reference_mode!r} (only 'block_mean' runs without references)." + f"{list(wide_all.columns)}. The toggle specialist method needs at least one reference turbine " + f"in reference_mode {mode!r} (only 'block_mean' runs without references)." ) raise ValueError(msg) - # Narrow to the campaign's turbines and blank the cells it says may not contribute, so the - # estimate and every diagnostic below see one consistent selection. + # Narrow to the mode's turbines and blank the cells the campaign says may not contribute, so + # the estimate and every diagnostic see one consistent selection. ref_set = set(refs) - wide = mi.context.mask_invalid(wide[[c for c in wide.columns if c == test or c in ref_set]]) + wide = mi.context.mask_invalid(wide_all[[c for c in wide_all.columns if c == test or c in ref_set]]) - if self.columns.availability not in mi.scada_df.columns: - msg = ( - f"the availability column {self.columns.availability!r} (columns.availability) is not in " - f"scada_df; the downtime filter is required for the toggle specialist method and cannot be skipped." - ) - raise ValueError(msg) - - timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) - self._check_pairing_gap(timebase) - block = self._reference_block(timebase) - rows = resolve_toggle(mi.upgrade_timing, wide.index) baseline = rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) campaign = baseline | rows.upgraded ids = ( block_ids(wide.index, start=wide.index[campaign].min(), block=block) - if block is not None and campaign.any() + if mode != "sum" and block is not None and campaign.any() else None ) selection = self._selection(mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids) used = selection.blocked.to_numpy() accounting = self._selection_accounting(selection, rows=rows) - ref_total = self._reference_total(wide=wide, refs=refs, test_pw=test_pw, used=used, ids=ids) + ref_total = self._reference_total(mode=mode, wide=wide, refs=refs, test_pw=test_pw, used=used, ids=ids) rho_base = _rho(test_pw, ref_total, used & baseline) rho_up = _rho(test_pw, ref_total, used & rows.upgraded) @@ -355,45 +445,22 @@ def estimate(self, mi: MethodInput) -> MethodOutput: baseline=used & baseline, used=used, ) - - stats = _segment_stats( - mi, - wide=wide, - used=used, - toggle_rows=rows, - ref_total=ref_total, - timebase=timebase, - active_power_col=self.columns.active_power, - ) - sigma_overall = _cell_sigma(boot, _OVERALL) - self._write_outputs( - mi, - wide=wide, - stats=stats, + diagnostics.insert(0, "component", mode) + return _Estimate( + mode=mode, + refs=refs, + selection=selection, used=used, ref_total=ref_total, + label=rho_label * ref_total, rho_base=rho_base, rho_up=rho_up, uplift=uplift, - sigma_overall=sigma_overall, - n_refs=len(refs), - refs=refs, - timebase=timebase, + sigma_overall=_cell_sigma(boot, _OVERALL), per_bin=per_bin, - diagnostics=diagnostics, + boot=boot, accounting=accounting, - selection=selection, - rows=rows, - ) - return MethodOutput( - p50_overall=float(uplift), - p50_by_condition=per_bin, - sigma_overall=sigma_overall, - uncertainty_diagnostics=diagnostics, - labeled_rows=self._labeled_rows( - mi, wide=wide, test=test, used=used, rows=rows, rho_label=rho_label, ref_total=ref_total - ), - selection_accounting=accounting, + diagnostics=diagnostics, ) def _labeled_rows( @@ -404,8 +471,7 @@ def _labeled_rows( test: str, used: npt.NDArray[np.bool_], rows: ToggleRowSets, - rho_label: float, - ref_total: npt.NDArray[np.float64], + label: npt.NDArray[np.float64], ) -> pd.DataFrame: """Return the test turbine's own records, tagged with the labels this estimate was built from. @@ -425,10 +491,10 @@ def _on_test_rows(values: npt.NDArray[np.generic]) -> npt.NDArray[np.generic]: # The bin label is the same reference-derived baseline power the uplift binned on, so a row # cannot sit in one bin here and another there. Outside the outer edges pd.cut gives NaN, # which is carried through as "this row belongs to no bin" rather than clipped to an edge. - if "power" in self.conditions and np.isfinite(rho_label): + if "power" in self.conditions and np.isfinite(label).any(): assert self.rated_power_kw is not None # noqa: S101 - guaranteed by __post_init__ bins = condition_bins("power", rated_power_kw=self.rated_power_kw) - labeled["power_bin"] = _on_test_rows(np.asarray(pd.cut(rho_label * ref_total, bins=bins))) + labeled["power_bin"] = _on_test_rows(np.asarray(pd.cut(label, bins=bins))) return labeled def _cell_membership( @@ -539,6 +605,7 @@ def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta | None: if self.reference_mode == "sum": return None block = self.reference_block if self.reference_block is not None else self.pairing_max_gap + # ``combined`` needs the block for its block_mean component. if block is None: msg = ( f"{self.name}: reference_mode {self.reference_mode!r} needs a block length: set reference_block " @@ -557,25 +624,26 @@ def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta | None: def _reference_total( self, *, + mode: str, wide: pd.DataFrame, refs: list[str], test_pw: npt.NDArray[np.float64], used: npt.NDArray[np.bool_], ids: npt.NDArray[np.int64] | None, ) -> npt.NDArray[np.float64]: - """Return the per-row reference power for ``reference_mode``; NaN on rows no surviving block covers. + """Return the per-row reference power for ``mode``; NaN on rows no surviving block covers. Every used row of a block gets the same block-level factor, so within a block the on/off ratio cancels the block's wind level whatever the on/off count mix (the block mean carries the uplift at ``n_on / n`` strength in both denominators, a second-order effect). """ ref_sum = wide[refs].sum(axis=1).to_numpy(dtype=float) if refs else np.zeros(len(wide)) - if self.reference_mode == "sum": + if mode == "sum": return ref_sum if ids is None or not used.any(): return np.full(len(wide), np.nan) block_test = _block_sum(ids, test_pw, used) - if self.reference_mode == "block_mean": + if mode == "block_mean": n_used = pd.Series(used.astype(float)).groupby(ids).sum() return _per_row(ids, block_test / n_used.reindex(block_test.index)) scale = block_test / _block_sum(ids, ref_sum, used) @@ -730,20 +798,10 @@ def _write_outputs( *, wide: pd.DataFrame, stats: pd.DataFrame, - used: np.ndarray, - ref_total: npt.NDArray[np.float64], - rho_base: float, - rho_up: float, - uplift: float, - sigma_overall: float, - n_refs: int, - refs: list[str], + est: _Estimate, + components: dict[str, _Estimate], timebase: pd.Timedelta, - per_bin: pd.DataFrame | None = None, - diagnostics: pd.DataFrame | None = None, - accounting: pd.DataFrame | None = None, - selection: _Selection | None = None, - rows: ToggleRowSets | None = None, + rows: ToggleRowSets, ) -> None: """Write the data-stats CSV, the headline results CSV, the per-bin CSV and (optionally) the plots.""" upgrade_start = toggle_upgrade_start(mi.upgrade_timing, wide.index) @@ -766,12 +824,14 @@ def _write_outputs( "test_wtg": mi.test_wtg, "mode": "toggle", "n_turbines": wide.shape[1], - "n_refs": n_refs, + "n_refs": len(est.refs), "reference_mode": self.reference_mode, - "ratio_baseline": rho_base, - "ratio_upgraded": rho_up, - "uplift_frc": uplift, - "uplift_sigma_frc": sigma_overall, + "ratio_baseline": est.rho_base, + "ratio_upgraded": est.rho_up, + "uplift_frc": est.uplift, + "uplift_sigma_frc": est.sigma_overall, + **{f"uplift_frc_{mode}": component.uplift for mode, component in components.items()}, + **{f"uplift_sigma_frc_{mode}": component.sigma_overall for mode, component in components.items()}, "block_hours": self.block_hours, "n_resamples": self.n_resamples, "n_used_timestamps_baseline": used_base, @@ -782,12 +842,13 @@ def _write_outputs( ) results.to_csv(run_dir / f"{run_name}_results_{ts}.csv", index=False) - if per_bin is not None: - per_bin.to_csv(run_dir / f"{run_name}_by_power_bin_{ts}.csv", index=False) - if diagnostics is not None: - diagnostics.to_csv(run_dir / f"{run_name}_uncertainty_{ts}.csv", index=False) - if accounting is not None: - accounting.to_csv(run_dir / f"{run_name}_selection_{ts}.csv", index=False) + if est.per_bin is not None: + est.per_bin.to_csv(run_dir / f"{run_name}_by_power_bin_{ts}.csv", index=False) + est.diagnostics.to_csv(run_dir / f"{run_name}_uncertainty_{ts}.csv", index=False) + # Every component's row selection, so a blend's two selections can be told apart. + pd.concat( + [component.accounting.assign(component=mode) for mode, component in components.items()], ignore_index=True + ).to_csv(run_dir / f"{run_name}_selection_{ts}.csv", index=False) if self.save_plots: _save_plots( @@ -795,31 +856,30 @@ def _write_outputs( wide=wide, mi=mi, test=mi.test_wtg, - used=used, - ref_total=ref_total, - reference_mode=self.reference_mode, + used=est.used, + ref_total=est.ref_total, + reference_mode=est.mode, timebase=timebase, active_power_col=self.columns.active_power, ) - if per_bin is not None: + if est.per_bin is not None: _save_per_bin_plot( run_dir / "plots" / stages.CONDITIONAL_UPLIFT / f"{mi.test_wtg}_per_bin_uplift.png", - per_bin=per_bin, + per_bin=est.per_bin, test=mi.test_wtg, active_power_col=self.columns.active_power, ) - if selection is not None and rows is not None: - _save_pairing_gap_plot( - run_dir / "plots" / stages.FILTER / f"{mi.test_wtg}_pairing_gap.png", - index=wide.index, - selection=selection, - rows=rows, - timebase=timebase, - max_gap=self.pairing_max_gap, - test=mi.test_wtg, - ) + _save_pairing_gap_plot( + run_dir / "plots" / stages.FILTER / f"{mi.test_wtg}_pairing_gap.png", + index=wide.index, + selection=est.selection, + rows=rows, + timebase=timebase, + max_gap=self.pairing_max_gap, + test=mi.test_wtg, + ) self._write_shared_diagnostics( - mi, run_dir=run_dir, wide=wide, used=used, timebase=timebase, turbines=[mi.test_wtg, *refs] + mi, run_dir=run_dir, wide=wide, used=est.used, timebase=timebase, turbines=[mi.test_wtg, *est.refs] ) def _write_shared_diagnostics( @@ -886,6 +946,82 @@ def _per_bin_counterfactual( return rho_row * ref_total +def _combine(a: _Estimate, b: _Estimate) -> _Estimate: + """Blend two component estimates by inverse variance, headline and per bin. + + The blend's used rows are the union of the components' (a row contributed to at least one + estimate); its bin label is ``a``'s where ``a`` used the row and ``b``'s otherwise, so the two + components' bins, which share edges, are merged by name. Row-selection accounting, ``rho`` and + the reference-side diagnostics are ``a``'s (the reference-based component); ``b``'s selection + is written to the selection CSV. The diagnostics carry both components' rows and a ``combined`` + row per cell with the blend's sigma, the weight on ``a`` and the correlation it used. + """ + + def _resamples(est: _Estimate, cell: str) -> npt.NDArray[np.float64]: + return est.boot.resamples.get(cell, np.array([])) if est.boot is not None else np.array([]) + + def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> CombinedEstimate: + return combine_estimates((est_a, sig_a, _resamples(a, cell)), (est_b, sig_b, _resamples(b, cell))) + + overall = _cell(_OVERALL, a.uplift, a.sigma_overall, b.uplift, b.sigma_overall) + blended_cells = {_OVERALL: overall} + + per_bin = None + if a.per_bin is not None and b.per_bin is not None: + fa = a.per_bin.set_index(a.per_bin["condition_bin"].astype(str)) + fb = b.per_bin.set_index(b.per_bin["condition_bin"].astype(str)) + records = [] + for cell in fa.index: + ra, rb = fa.loc[cell], fb.loc[cell] + blend = _cell(cell, ra["p50_uplift"], ra["sigma_uplift"], rb["p50_uplift"], rb["sigma_uplift"]) + blended_cells[cell] = blend + take = ra if np.isfinite(ra["p50_uplift"]) else rb + records.append( + { + "condition": ra["condition"], + "condition_bin": ra["condition_bin"], + "p50_uplift": blend.estimate, + "n_records": int(max(ra["n_records"], rb["n_records"])), + "sum_actual": take["sum_actual"], + "sum_counterfactual": take["sum_counterfactual"], + "sigma_uplift": blend.sigma, + } + ) + per_bin = pd.DataFrame(records) + + combined_rows = pd.DataFrame( + [ + { + "component": "combined", + "condition": _OVERALL if cell == _OVERALL else "power", + "condition_bin": cell, + "sigma": blend.sigma, + f"weight_{a.mode}": blend.weight_a, + "correlation": blend.correlation, + } + for cell, blend in blended_cells.items() + ] + ) + diagnostics = pd.concat([a.diagnostics, b.diagnostics, combined_rows], ignore_index=True) + + return _Estimate( + mode="combined", + refs=a.refs, + selection=a.selection, + used=a.used | b.used, + ref_total=a.ref_total, + label=np.where(a.used, a.label, b.label), + rho_base=a.rho_base, + rho_up=a.rho_up, + uplift=overall.estimate, + sigma_overall=overall.sigma, + per_bin=per_bin, + boot=None, + accounting=a.accounting, + diagnostics=diagnostics, + ) + + def _cell_sigma(boot: BootstrapResult | None, cell: str) -> float: """Return one cell's 1-sigma, or NaN when the bootstrap did not run or never saw that cell.""" if boot is None or cell not in boot.cells: @@ -922,6 +1058,7 @@ def _uncertainty_diagnostics( "n_upgraded_records": int((member & up_used).sum()), "n_baseline_records": int((member & base_used).sum()), "n_blocks": boot.n_blocks if boot is not None else 0, + "sigma": cell_boot.sigma if cell_boot is not None else nan, # Both components, not just the reported max: a blend rule can then be re-judged from # a saved sweep rather than by re-running one. "sigma_bootstrap": cell_boot.sigma_bootstrap if cell_boot is not None else nan, diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index 2e30c95a..cb0a602d 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -12,7 +12,7 @@ import pandas as pd import pytest -from benchmarking.baselines.block_bootstrap import bootstrap_ratio_uplift, relative_scatter +from benchmarking.baselines.block_bootstrap import bootstrap_ratio_uplift, combine_estimates, relative_scatter _TIMEBASE = pd.Timedelta(minutes=10) @@ -380,3 +380,70 @@ def test_sigma_scales_down_smoothly_as_noise_falls(self) -> None: sigmas.append(_run(case).cells["overall"].sigma) assert sigmas == sorted(sigmas, reverse=True) assert sigmas[-1] < sigmas[0] / 10, "no floor is arresting the descent" + + +class TestResamplesAreExposed: + def test_each_cell_carries_its_resampled_uplifts(self) -> None: + result = _run(_case(), n_resamples=300) + samples = result.resamples["overall"] + assert samples.shape == (300,) + assert np.std(samples[np.isfinite(samples)], ddof=1) == pytest.approx(result.cells["overall"].sigma_bootstrap) + + def test_no_records_means_no_resamples(self) -> None: + empty = { + k: np.array([], dtype=bool if k in ("upgraded", "baseline") else float) + for k in ("test_power", "ref_total", "upgraded", "baseline") + } + result = bootstrap_ratio_uplift( + times=_timeline(0), + cell_membership={"overall": np.array([], dtype=bool)}, + campaign_start=pd.Timestamp("2020-01-01", tz="UTC"), + campaign_end=pd.Timestamp("2020-01-08", tz="UTC"), + timebase=_TIMEBASE, + block_hours=48.0, + n_resamples=100, + seed=0, + **empty, + ) + assert result.resamples == {} + + +class TestCombineEstimates: + """Inverse-variance blend of two estimates of the same quantity, honouring their resample correlation.""" + + @staticmethod + def _samples(rho: float, *, n: int = 2000, seed: int = 0) -> tuple[np.ndarray, np.ndarray]: + rng = np.random.default_rng(seed) + a = rng.standard_normal(n) + b = rho * a + np.sqrt(1.0 - rho**2) * rng.standard_normal(n) + return a, b + + def test_independent_components_give_the_naive_inverse_variance_sigma(self) -> None: + a, b = self._samples(0.0) + out = combine_estimates((0.02, 1.0, a), (0.04, 2.0, b)) + assert out.weight_a == pytest.approx(0.8) + assert out.estimate == pytest.approx(0.8 * 0.02 + 0.2 * 0.04) + assert out.sigma == pytest.approx(1.0 / np.sqrt(1.0 + 0.25), abs=0.03) + + def test_a_correlation_widens_the_combined_sigma(self) -> None: + a, b = self._samples(0.8) + out = combine_estimates((0.0, 1.0, a), (0.0, 1.0, b)) + assert out.correlation == pytest.approx(0.8, abs=0.03) + assert out.sigma == pytest.approx(np.sqrt((1 + 0.8) / 2), abs=0.03) + + def test_one_unusable_component_leaves_the_other_untouched(self) -> None: + a, b = self._samples(0.0) + out = combine_estimates((float("nan"), float("nan"), a), (0.03, 2.0, b)) + assert out.weight_a == 0.0 + assert out.estimate == 0.03 + assert out.sigma == 2.0 + + def test_neither_usable_is_nan(self) -> None: + out = combine_estimates((float("nan"), float("nan"), np.array([])), (float("nan"), float("nan"), np.array([]))) + assert np.isnan(out.estimate) + assert np.isnan(out.sigma) + + def test_missing_resamples_assume_a_half_correlation(self) -> None: + out = combine_estimates((0.0, 1.0, np.array([])), (0.0, 1.0, np.array([]))) + assert out.correlation == 0.5 + assert out.sigma == pytest.approx(np.sqrt((1 + 0.5) / 2)) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index d705af53..fd85b5ef 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1363,3 +1363,47 @@ def test_a_block_mode_without_a_block_length_raises(self) -> None: scada, schedule = _block_case(n_blocks=10, ref_scale=np.ones(10)) with pytest.raises(ValueError, match="reference_block"): _estimate(scada, schedule, reference_mode="block_scaled") + + +class TestCombinedMode: + """``reference_mode="combined"``: inverse-variance blend of the ``sum`` and ``block_mean`` estimates.""" + + def test_recovers_the_uplift_when_both_components_are_exact(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + out = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + + def test_falls_back_to_block_mean_when_the_references_are_out(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5), seed=3) + scada.loc[(scada[_TURBINE_COL] == "R1").to_numpy(), _POWER_COL] = np.nan + combined = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) + alone = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + assert combined.p50_overall == pytest.approx(alone.p50_overall) + assert combined.sigma_overall == pytest.approx(alone.sigma_overall) + + def test_sigma_lies_between_the_naive_blend_and_the_better_component(self) -> None: + scada, schedule = _noisy_toggle_case() + gap = pd.Timedelta(minutes=20) + combined = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=gap) + total = _estimate(scada, schedule, reference_mode="sum", pairing_max_gap=gap) + block = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=gap) + naive = 1.0 / np.sqrt(1.0 / total.sigma_overall**2 + 1.0 / block.sigma_overall**2) + assert naive <= combined.sigma_overall <= max(total.sigma_overall, block.sigma_overall) + + def test_diagnostics_name_the_components(self) -> None: + scada, schedule = _noisy_toggle_case(n=400) + out = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=pd.Timedelta(minutes=20)) + assert set(out.uncertainty_diagnostics["component"]) == {"sum", "block_mean", "combined"} + combined = out.uncertainty_diagnostics[out.uncertainty_diagnostics["component"] == "combined"] + assert np.isfinite(combined.loc[combined["condition_bin"] == "overall", "weight_sum"]).all() + + def test_used_rows_are_the_union_of_the_components(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() + scada.loc[is_ref & (np.arange(len(scada)) % 7 == 0), _POWER_COL] = np.nan + out = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) + total = _estimate(scada, schedule, reference_mode="sum", reference_block=_BLOCK) + used_combined = out.labeled_rows["used"] == True # noqa: E712 - NaN (absent timestamp) must read as unused + used_sum = total.labeled_rows["used"] == True # noqa: E712 + assert used_combined.sum() > used_sum.sum() + assert used_combined.all() From 6f635ecd995bbd5850b52c7d7c43d64285c223bd Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 13:41:57 +0100 Subject: [PATCH 05/18] Report NaN, not float residue, for a bootstrap cell too sparse to fall back on A cell with at most three records a side blends to the fallback alone; when the per-record scatter cannot be measured either, the result is NaN rather than the ~1e-17 std of resamples that all see the same handful of records. That residue read as near-certainty, and in the combined reference mode it took the whole weight. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 9 ++++++- .../baselines/test_block_bootstrap.py | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 2b3bc308..389902a5 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -402,7 +402,14 @@ def _summarise( def _blend(*, bootstrap: float, fallback: float, weight: float) -> float: - """Linear ramp ``weight*bootstrap + (1-weight)*fallback``, using whichever side is finite.""" + """Linear ramp ``weight*bootstrap + (1-weight)*fallback``, using whichever side is finite. + + A cell at the sparse end (``weight == 0``) reports the fallback alone, even when that is NaN: + with so few records every resample that sees the cell is the same handful, so the bootstrap's + std is float residue (~1e-17), which would read as near-certainty where there is none. + """ + if weight == 0.0: + return fallback if math.isfinite(bootstrap) and math.isfinite(fallback): return weight * bootstrap + (1.0 - weight) * fallback if math.isfinite(bootstrap): diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index cb0a602d..8cb9c48b 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -447,3 +447,30 @@ def test_missing_resamples_assume_a_half_correlation(self) -> None: out = combine_estimates((0.0, 1.0, np.array([])), (0.0, 1.0, np.array([]))) assert out.correlation == 0.5 assert out.sigma == pytest.approx(np.sqrt((1 + 0.5) / 2)) + + +class TestASparseCellWithNoFallbackReportsNaN: + """Two records in a long campaign: every resample that sees them is identical, so the bootstrap's + std is float residue (~1e-17), and the per-record scatter cannot be measured either. That is no + uncertainty, not a tiny one.""" + + def test_two_records_report_nan_rather_than_float_residue(self) -> None: + times = _timeline(2016) + keep = np.zeros(2016, dtype=bool) + keep[[1000, 1001]] = True + on = np.zeros(2016, dtype=bool) + on[1000] = True + case = { + "times": times[keep], + "test_power": np.array([800.0, 750.0]), + "ref_total": np.array([1000.0, 1000.0]), + "upgraded": on[keep], + "baseline": ~on[keep], + "cell_membership": {"overall": np.ones(2, dtype=bool)}, + "campaign_start": times[0], + "campaign_end": times[-1], + "timebase": _TIMEBASE, + } + cell = _run(case).cells["overall"] + assert np.isnan(cell.sigma_fallback) + assert np.isnan(cell.sigma) From 5fe8d39e4fe62ffe5d7ade4a3f1ba8225dd0ce46 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 14:15:12 +0100 Subject: [PATCH 06/18] Refuse to blend components whose bootstrap resamples cannot be paired draw by draw Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 7 +++++++ benchmarking/baselines/toggle_specialist.py | 5 +++++ tests/benchmarking/baselines/test_block_bootstrap.py | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 389902a5..b187df0f 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -137,6 +137,13 @@ def combine_estimates( weight_a = wa / (wa + wb) weight_b = 1.0 - weight_a + if len(res_a) and len(res_b) and len(res_a) != len(res_b): + msg = ( + f"the two components' resample arrays have different lengths ({len(res_a)} vs {len(res_b)}); " + f"they can only be paired draw by draw when both bootstraps ran with the same campaign span, " + f"timebase, block length, resample count and seed." + ) + raise ValueError(msg) n_pairs = min(len(res_a), len(res_b)) paired = np.isfinite(res_a[:n_pairs]) & np.isfinite(res_b[:n_pairs]) if paired.sum() >= _MIN_RESAMPLES_FOR_SPREAD: diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 5241fe4e..78b785a0 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -956,6 +956,11 @@ def _combine(a: _Estimate, b: _Estimate) -> _Estimate: is written to the selection CSV. The diagnostics carry both components' rows and a ``combined`` row per cell with the blend's sigma, the weight on ``a`` and the correlation it used. """ + if a.boot is not None and b.boot is not None and a.boot.n_blocks != b.boot.n_blocks: + # Both bootstraps come from the same method instance (same span, timebase, block length, + # resample count and seed), so their draws pair up; a block-count mismatch means they do not. + msg = f"component bootstraps drew different block counts ({a.boot.n_blocks} vs {b.boot.n_blocks})" + raise ValueError(msg) def _resamples(est: _Estimate, cell: str) -> npt.NDArray[np.float64]: return est.boot.resamples.get(cell, np.array([])) if est.boot is not None else np.array([]) diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index 8cb9c48b..c6b55fea 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -474,3 +474,11 @@ def test_two_records_report_nan_rather_than_float_residue(self) -> None: cell = _run(case).cells["overall"] assert np.isnan(cell.sigma_fallback) assert np.isnan(cell.sigma) + + +class TestCombineEstimatesRefusesUnpairedResamples: + """The correlation only means something when the two resample arrays came from the same block draws.""" + + def test_different_resample_counts_raise(self) -> None: + with pytest.raises(ValueError, match="resample"): + combine_estimates((0.0, 1.0, np.zeros(100)), (0.0, 1.0, np.zeros(99))) From c4bc9d790e0c2013df2aaee9571774431bb59705 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 15:25:37 +0100 Subject: [PATCH 07/18] Blend correlated components with minimum-variance weights so the blend is never worse than its better leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weights were plain inverse variance, with the resample correlation entering only the reported sigma; a noisier leg correlated with the better one could then widen the blend beyond the better leg alone. The weights are now wa ∝ max(sb² - r·sa·sb, 0) and wb ∝ max(sa² - r·sa·sb, 0), which drop a strongly correlated noisy leg to zero. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 27 ++++++++++--------- .../baselines/test_block_bootstrap.py | 19 +++++++++++-- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index b187df0f..83d6c41e 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -112,11 +112,13 @@ def combine_estimates( """Blend two ``(estimate, sigma, resamples)`` of the same quantity by inverse variance. A component is usable when both its estimate and its sigma are finite. With two usable - components the weights are ``1 / sigma**2`` (a zero sigma takes all the weight) and the blend's - variance is ``(wa**2 sa**2 + wb**2 sb**2 + 2 wa wb r sa sb) / (wa + wb)**2`` with ``r`` the - Pearson correlation of the draw-paired finite resamples, or - :data:`_DEFAULT_COMPONENT_CORRELATION` when fewer than two pairs exist. One usable component is - returned as is; none gives NaN throughout. + components the weights are the minimum-variance ones for two correlated estimates, + ``wa ∝ max(sb² - r sa sb, 0)`` and ``wb ∝ max(sa² - r sa sb, 0)`` (a zero sigma takes all the + weight; both weights zero splits evenly), with ``r`` the Pearson correlation of the draw-paired + finite resamples, or :data:`_DEFAULT_COMPONENT_CORRELATION` when fewer than two pairs exist. + The blend's variance is ``wa² sa² + wb² sb² + 2 wa wb r sa sb``, which these weights keep at or + below the better component's: a noisier leg correlated at ``r >= sb / sa`` gets no weight rather + than widening the blend. One usable component is returned as is; none gives NaN throughout. """ est_a, sig_a, res_a = a est_b, sig_b, res_b = b @@ -130,13 +132,6 @@ def combine_estimates( if not (usable_a and usable_b): return CombinedEstimate(estimate=nan, sigma=nan, weight_a=nan, correlation=nan) - if sig_a == 0.0 or sig_b == 0.0: - weight_a = 0.5 if sig_a == sig_b else float(sig_a == 0.0) - else: - wa, wb = 1.0 / sig_a**2, 1.0 / sig_b**2 - weight_a = wa / (wa + wb) - weight_b = 1.0 - weight_a - if len(res_a) and len(res_b) and len(res_a) != len(res_b): msg = ( f"the two components' resample arrays have different lengths ({len(res_a)} vs {len(res_b)}); " @@ -151,6 +146,14 @@ def combine_estimates( correlation = r if math.isfinite(r) else _DEFAULT_COMPONENT_CORRELATION else: correlation = _DEFAULT_COMPONENT_CORRELATION + + if sig_a == 0.0 or sig_b == 0.0: + weight_a = 0.5 if sig_a == sig_b else float(sig_a == 0.0) + else: + cross = correlation * sig_a * sig_b + wa, wb = max(sig_b**2 - cross, 0.0), max(sig_a**2 - cross, 0.0) + weight_a = 0.5 if wa + wb == 0.0 else wa / (wa + wb) + weight_b = 1.0 - weight_a variance = weight_a**2 * sig_a**2 + weight_b**2 * sig_b**2 + 2.0 * weight_a * weight_b * correlation * sig_a * sig_b return CombinedEstimate( estimate=weight_a * est_a + weight_b * est_b, diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index c6b55fea..91c9dfa9 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -421,8 +421,9 @@ def _samples(rho: float, *, n: int = 2000, seed: int = 0) -> tuple[np.ndarray, n def test_independent_components_give_the_naive_inverse_variance_sigma(self) -> None: a, b = self._samples(0.0) out = combine_estimates((0.02, 1.0, a), (0.04, 2.0, b)) - assert out.weight_a == pytest.approx(0.8) - assert out.estimate == pytest.approx(0.8 * 0.02 + 0.2 * 0.04) + # The sampled correlation is ~0, not exactly 0, so the weights land near the naive 0.8 / 0.2. + assert out.weight_a == pytest.approx(0.8, abs=0.02) + assert out.estimate == pytest.approx(0.8 * 0.02 + 0.2 * 0.04, abs=1e-3) assert out.sigma == pytest.approx(1.0 / np.sqrt(1.0 + 0.25), abs=0.03) def test_a_correlation_widens_the_combined_sigma(self) -> None: @@ -431,6 +432,20 @@ def test_a_correlation_widens_the_combined_sigma(self) -> None: assert out.correlation == pytest.approx(0.8, abs=0.03) assert out.sigma == pytest.approx(np.sqrt((1 + 0.8) / 2), abs=0.03) + def test_a_correlated_noisier_component_drops_out_rather_than_widening_the_blend(self) -> None: + # Minimum-variance weights for correlated estimates: with r >= sb/sa the noisier leg gets no weight. + a, b = self._samples(0.8) + out = combine_estimates((0.10, 2.0, 2.0 * a), (0.02, 1.0, b)) + assert out.weight_a == 0.0 + assert out.estimate == pytest.approx(0.02) + assert out.sigma == pytest.approx(1.0) + + def test_the_blend_is_never_worse_than_its_better_component(self) -> None: + for rho in (0.0, 0.3, 0.6, 0.9): + a, b = self._samples(rho, seed=1) + out = combine_estimates((0.0, 1.5, 1.5 * a), (0.0, 1.0, b)) + assert out.sigma <= 1.0 + 1e-9, rho + def test_one_unusable_component_leaves_the_other_untouched(self) -> None: a, b = self._samples(0.0) out = combine_estimates((float("nan"), float("nan"), a), (0.03, 2.0, b)) From 469819dbbc73a7e529354c728e82e7478409e75c Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 15:26:20 +0100 Subject: [PATCH 08/18] Run the same normal-operation filter on every turbine the toggle specialist uses The test turbine went through NormalOperationFilter while the references got a separate complete-case plus availability check. With the stuck filter off both tests are the same, so nothing changes today; one code path means any future filter change reaches the references. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/toggle_specialist.py | 45 +++++++++------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 78b785a0..ea881bfe 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -738,34 +738,27 @@ def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) - def _filtered_mask( self, mi: MethodInput, *, wide: pd.DataFrame, test: str, refs: list[str], timebase: pd.Timedelta ) -> pd.Series: - """Complete-case timestamps that also pass downtime filtering on the test turbine and every reference. - - Returns a bool Series on ``wide.index``. Every turbine (test and references) must be - available (counter >= a full period) and have finite power — a down turbine on either side - of the ratio is therefore excluded. The test turbine additionally goes through the shared - :class:`NormalOperationFilter` (the same downtime + finite-power logic the power model uses; - the stuck filter is left off here as the ratio sums raw power rather than fitting a model). + """Timestamps at which the test turbine and every reference pass the same normal-operation filter. + + Returns a bool Series on ``wide.index``. One :class:`NormalOperationFilter` (the downtime + + finite-power logic the power model uses; the stuck filter is left off here as the ratio sums + raw power rather than fitting a model) is applied to each turbine's own rows, so a turbine + that is down or has no power on either side of the ratio excludes the timestamp. The same + filter on every turbine means any future change to it (e.g. the stuck filter) applies to + the references exactly as to the test turbine. """ - turbines = [test, *refs] - complete = wide[turbines].notna().all(axis=1) - - full = timebase.total_seconds() - avail = _wide_column(mi.scada_df, turbine_col=mi.turbine_col, value_col=self.columns.availability).reindex( - index=wide.index, columns=turbines - ) - all_available = (avail >= full).all(axis=1) - - test_rows = mi.scada_df[mi.scada_df[mi.turbine_col] == test] - test_keep = ( - NormalOperationFilter( - active_power_col=self.columns.active_power, - availability_col=self.columns.availability, - apply_stuck_filter=False, - ) - .keep_mask(test_rows, timebase=timebase) - .reindex(wide.index, fill_value=False) + normal = NormalOperationFilter( + active_power_col=self.columns.active_power, + availability_col=self.columns.availability, + apply_stuck_filter=False, ) - return complete & all_available & test_keep + keep = pd.Series(data=True, index=wide.index, dtype=bool) + for turbine in (test, *refs): + rows = mi.scada_df[mi.scada_df[mi.turbine_col] == turbine] + keep &= normal.keep_mask(rows, timebase=timebase).reindex(wide.index, fill_value=False) + # ``wide`` is the masked pivot: a cell the campaign context blanked must not count either. + keep &= wide[turbine].notna() + return keep def _excluded(self, mi: MethodInput, *, turbines: list[str], index: pd.DatetimeIndex) -> pd.Series: """Boolean mask on *index*: timestamps at which any of ``turbines`` carries a caller-set exclusion flag. From 0d11a51a233b1ac8840fd81a5870c2dc0230aad0 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 15:30:21 +0100 Subject: [PATCH 09/18] Add an exclude_block column role: a flagged row removes its whole reference block The pairing filter removes stretches where one toggle state is absent, never the counterpart of a single dropped row, so a caller-side selection that thins one state's rows in particular conditions (a controlled-data residual test, say) leaves the two states with different wind mixes. A row flagged in the exclude_block column drops every row of its one-cycle block, for every turbine and in every reference mode, so both states lose the same conditions. Block ids are now computed whenever a block length is available, sum mode included. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/toggle_specialist.py | 72 ++++++++++++------- benchmarking/synthetic/schema.py | 5 ++ .../baselines/test_toggle_specialist.py | 60 ++++++++++++++++ 3 files changed, 113 insertions(+), 24 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index ea881bfe..147658f0 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -15,7 +15,10 @@ An optional **pairing filter** (``pairing_max_gap``) then drops any used row with no used row of the other segment nearby, so a filter that removes one segment's rows in some conditions also removes the other segment's rows from those conditions. Rows kept per segment after each selection stage are -reported as ``MethodOutput.selection_accounting`` and written to a selection CSV. +reported as ``MethodOutput.selection_accounting`` and written to a selection CSV. A caller may also +flag rows through ``columns.exclude_block``: such a row removes its whole reference block (one +toggle cycle), so both states lose the same conditions — a symmetric alternative to ``exclude_row`` +for a selection that would otherwise thin one state's rows in particular conditions. **Reference modes** (``reference_mode``) decide what the test power is compared against. ``"sum"`` is the summed power of the references. The two *block* modes tile the campaign into fixed @@ -259,8 +262,8 @@ class ToggleSpecialistMethod: stage differ by more than this. :param reference_mode: one of :data:`REFERENCE_MODES` (see the module docstring). ``"block_mean"`` uses no references: they do not enter the row filters either, and none need exist. - :param reference_block: block length for the block modes, a positive whole multiple of the - timebase; defaults to ``pairing_max_gap`` (one cycle). Ignored in ``"sum"`` mode. + :param reference_block: block length for the block modes and for ``columns.exclude_block``, a + positive whole multiple of the timebase; defaults to ``pairing_max_gap`` (one cycle). """ columns: ColumnSchema @@ -393,10 +396,12 @@ def _component( campaign = baseline | rows.upgraded ids = ( block_ids(wide.index, start=wide.index[campaign].min(), block=block) - if mode != "sum" and block is not None and campaign.any() + if block is not None and campaign.any() else None ) - selection = self._selection(mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids) + selection = self._selection( + mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids, both_states=mode != "sum" + ) used = selection.blocked.to_numpy() accounting = self._selection_accounting(selection, rows=rows) ref_total = self._reference_total(mode=mode, wide=wide, refs=refs, test_pw=test_pw, used=used, ids=ids) @@ -601,12 +606,15 @@ def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: raise ValueError(msg) def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta | None: - """Return the block length the block modes tile the campaign with; ``None`` in ``sum`` mode.""" - if self.reference_mode == "sum": - return None + """Return the block length (reference_block, else pairing_max_gap); ``None`` when neither is set. + + The block modes (and ``combined``, for its block_mean component) cannot run without one and + raise; ``sum`` mode only needs it to honour ``columns.exclude_block``. + """ block = self.reference_block if self.reference_block is not None else self.pairing_max_gap - # ``combined`` needs the block for its block_mean component. if block is None: + if self.reference_mode == "sum": + return None msg = ( f"{self.name}: reference_mode {self.reference_mode!r} needs a block length: set reference_block " f"or pairing_max_gap (one toggle cycle is the natural choice)." @@ -659,14 +667,18 @@ def _selection( timebase: pd.Timedelta, rows: ToggleRowSets, ids: npt.NDArray[np.int64] | None = None, + both_states: bool = True, ) -> _Selection: """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``. - ``ids`` are the reference-block ids (``None`` outside the block modes, or with no campaign - rows to tile); the ``block`` stage drops the rows of a block that lacks either state. + ``ids`` are the reference-block ids (``None`` with no block length, or no campaign rows to + tile). The ``block`` stage drops every row of a block that carries a ``columns.exclude_block`` + flag on any of the turbines (all modes) or, when ``both_states`` (the block modes), lacks + either state. """ + turbines = [test, *refs] filtered = self._filtered_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase) - not_excluded = filtered & ~self._excluded(mi, turbines=[test, *refs], index=wide.index) + not_excluded = filtered & ~self._excluded(mi, turbines=turbines, index=wide.index) in_segment = rows.campaign_baseline | rows.upgraded kept = not_excluded.to_numpy() if self.pairing_max_gap is not None: @@ -678,7 +690,16 @@ def _selection( ) kept = kept & (~in_segment | paired.baseline | paired.upgraded) paired_series = pd.Series(kept, index=wide.index) - if ids is not None: + block_flag = self._excluded(mi, turbines=turbines, index=wide.index, role="exclude_block").to_numpy() + if block_flag.any(): + if ids is None: + msg = ( + f"{self.name}: columns.exclude_block {self.columns.exclude_block!r} carries flags but there is " + f"no block to remove: set reference_block or pairing_max_gap (one toggle cycle)." + ) + raise ValueError(msg) + kept = kept & ~np.isin(ids, np.unique(ids[block_flag])) + if ids is not None and both_states: both = blocks_with_both_states(ids, baseline=kept & rows.campaign_baseline, upgraded=kept & rows.upgraded) kept = kept & (~in_segment | both) return _Selection( @@ -760,17 +781,19 @@ def _filtered_mask( keep &= wide[turbine].notna() return keep - def _excluded(self, mi: MethodInput, *, turbines: list[str], index: pd.DatetimeIndex) -> pd.Series: - """Boolean mask on *index*: timestamps at which any of ``turbines`` carries a caller-set exclusion flag. - - Reads the ``columns.exclude_row`` column of each listed turbine's rows (the test turbine and - the references the mode uses), so a flagged reference row removes the timestamp the way a - reference outage does. Absent column or unset role -> nothing excluded. Reindex fills missing - timestamps with ``False`` so an expanded index never becomes an exclusion. NaN raises rather - than coercing: ``astype(bool)`` reads a missing flag as ``True`` and drops the row, the - opposite of the safe default. + def _excluded( + self, mi: MethodInput, *, turbines: list[str], index: pd.DatetimeIndex, role: str = "exclude_row" + ) -> pd.Series: + """Boolean mask on *index*: timestamps at which any of ``turbines`` carries a caller-set ``role`` flag. + + Reads the ``columns.`` column (``exclude_row`` or ``exclude_block``) of each listed + turbine's rows (the test turbine and the references the mode uses), so a flagged reference + row counts the way a reference outage does. Absent column or unset role -> nothing flagged. + Reindex fills missing timestamps with ``False`` so an expanded index never becomes an + exclusion. NaN raises rather than coercing: ``astype(bool)`` reads a missing flag as ``True`` + and drops the row, the opposite of the safe default. """ - col = self.columns.exclude_row + col: str | None = getattr(self.columns, role) excluded = pd.Series(data=False, index=index, dtype=bool) if not col or col not in mi.scada_df.columns: return excluded @@ -778,7 +801,7 @@ def _excluded(self, mi: MethodInput, *, turbines: list[str], index: pd.DatetimeI flags = mi.scada_df.loc[mi.scada_df[mi.turbine_col] == turbine, col] if flags.isna().any(): msg = ( - f"exclude_row column {col!r} has {int(flags.isna().sum())} NaN value(s) for turbine " + f"{role} column {col!r} has {int(flags.isna().sum())} NaN value(s) for turbine " f"{turbine!r}; it must be boolean with no missing values (fill unknown rows with False explicitly)" ) raise ValueError(msg) @@ -912,6 +935,7 @@ def _write_shared_diagnostics( "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), "reference_mode": self.reference_mode, "reference_block": None if self.reference_block is None else str(self.reference_block), + "exclude_block_col": self.columns.exclude_block, } write_run_config(ctx, method_name=self.name, method_params=params) diff --git a/benchmarking/synthetic/schema.py b/benchmarking/synthetic/schema.py index 075280cb..f93ff79c 100644 --- a/benchmarking/synthetic/schema.py +++ b/benchmarking/synthetic/schema.py @@ -53,6 +53,10 @@ class ColumnSchema: default; when set, a method that honours it excludes the *test* turbine's flagged rows alongside the downtime filter. Must be all-``False`` where unknown (never NaN) so an expanded time index never turns a gap into an exclusion; a method honouring the role raises on NaN. + :param exclude_block: names a caller-supplied **boolean** column like ``exclude_row``, but a + flagged row removes the whole fixed-length block (one toggle cycle) around it, for every + turbine, rather than its own timestamp alone. For a selection that must stay symmetric + between toggle states (both lose the same conditions). Same never-NaN contract. """ turbine: str @@ -67,6 +71,7 @@ class ColumnSchema: nacelle_position: str | None = None ambient_temp: str | None = None exclude_row: str | None = None + exclude_block: str | None = None def northed(self, role: str) -> str: """Return the derived north-calibrated column name for ``role``, e.g. ``northed_YawAngleMean``. diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index fd85b5ef..c8182005 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1407,3 +1407,63 @@ def test_used_rows_are_the_union_of_the_components(self) -> None: used_sum = total.labeled_rows["used"] == True # noqa: E712 assert used_combined.sum() > used_sum.sum() assert used_combined.all() + + +_BLOCK_COL = "cycle_bad" +_BLOCK_COLUMNS = replace(_COLUMNS, exclude_block=_BLOCK_COL) + + +def _flag_block(scada: pd.DataFrame, *, turbine: str, at: pd.Timestamp) -> pd.DataFrame: + """Return a copy of ``scada`` with an all-False ``exclude_block`` column set on one ``turbine`` row.""" + out = scada.copy() + out[_BLOCK_COL] = False + out.loc[((out[_TURBINE_COL] == turbine) & (out.index == at)).to_numpy(), _BLOCK_COL] = True + return out + + +class TestExcludeBlock: + """``columns.exclude_block``: a flagged row removes its whole reference block, in every mode.""" + + @staticmethod + def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + return ToggleSpecialistMethod(columns=_BLOCK_COLUMNS, **kwargs).estimate( + MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) + ) + + def test_a_flagged_test_row_removes_its_whole_block_in_sum_mode(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + idx = pd.DatetimeIndex(pd.unique(scada.index)) + block_5 = idx[20:24] + out = self._run(_flag_block(scada, turbine="T1", at=block_5[1]), schedule, reference_block=_BLOCK) + assert out.labeled_rows is not None + used = out.labeled_rows["used"].astype(bool) + assert not used.loc[block_5].any() + assert used.drop(block_5).all() + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + acc = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + for segment in ("baseline", "upgraded"): + assert acc.loc[("block", segment)] == acc.loc[("pairing", segment)] - 2 + + def test_a_flagged_reference_row_removes_the_block_too(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + idx = pd.DatetimeIndex(pd.unique(scada.index)) + block_7 = idx[28:32] + out = self._run(_flag_block(scada, turbine="R1", at=block_7[3]), schedule, reference_block=_BLOCK) + assert out.labeled_rows is not None + used = out.labeled_rows["used"].astype(bool) + assert not used.loc[block_7].any() + assert used.drop(block_7).all() + + def test_a_flag_with_no_block_length_is_refused(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + idx = pd.DatetimeIndex(pd.unique(scada.index)) + with pytest.raises(ValueError, match="exclude_block"): + self._run(_flag_block(scada, turbine="T1", at=idx[21]), schedule) + + def test_no_flags_change_nothing(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + scada[_BLOCK_COL] = False + with_role = self._run(scada, schedule, reference_block=_BLOCK) + without = _estimate(scada.drop(columns=_BLOCK_COL), schedule, reference_block=_BLOCK) + assert with_role.p50_overall == pytest.approx(without.p50_overall) + assert with_role.sigma_overall == pytest.approx(without.sigma_overall) From 068cf806d49d3fc93a5276110dec16bb949ee165 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Mon, 5 Oct 2026 15:37:30 +0100 Subject: [PATCH 10/18] Add reference_choice='min_sigma': the sum estimate takes the reference subset with the smallest sigma Every non-empty subset of the available references is estimated in full (own row selection, bootstrap); the subset with the smallest finite headline sigma becomes the sum estimate, alone or as the blend's reference leg, ties to the larger subset. A reference that tracks badly or is screened out for long stretches loses to the subsets without it. Every subset's result is written to a reference_subsets CSV; the results CSV records the choice and the references used. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/toggle_specialist.py | 124 ++++++++++++++++-- .../baselines/test_toggle_specialist.py | 72 ++++++++++ 2 files changed, 184 insertions(+), 12 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 147658f0..ab814f22 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -34,6 +34,12 @@ Their errors come from different places (reference mismatch versus within-cycle drift), which is what the blend exploits. +**Reference choice** (``reference_choice``) decides which references the ``sum`` estimate (alone or +as the blend's reference leg) uses. ``"all"`` is every available reference; ``"min_sigma"`` runs the +``sum`` estimate on every non-empty subset of them and keeps the subset with the smallest headline +sigma (ties to the larger subset), so a reference that tracks badly, or is screened out for long +stretches, loses to the subsets without it. Every subset's result is written to a CSV. + Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the uplift, from the uplift's own frozen row selection and bin assignment, and only when the uplift is @@ -53,6 +59,7 @@ import logging import tempfile from dataclasses import dataclass, replace +from itertools import combinations from pathlib import Path from typing import TYPE_CHECKING, NamedTuple @@ -101,6 +108,10 @@ # The two estimates ``"combined"`` blends, in blend order (the first is the one the row-selection # accounting and the reference-side diagnostics report). _COMBINED_COMPONENTS: tuple[str, str] = ("sum", "block_mean") +# ``reference_choice`` values; see the module docstring. +REFERENCE_CHOICES: tuple[str, ...] = ("all", "min_sigma") +# ``min_sigma`` runs 2**n - 1 estimates; warn, rather than refuse, past this many references. +_MAX_REFS_FOR_SUBSETS = 6 logger = logging.getLogger(__name__) @@ -264,6 +275,8 @@ class ToggleSpecialistMethod: uses no references: they do not enter the row filters either, and none need exist. :param reference_block: block length for the block modes and for ``columns.exclude_block``, a positive whole multiple of the timebase; defaults to ``pairing_max_gap`` (one cycle). + :param reference_choice: one of :data:`REFERENCE_CHOICES` (see the module docstring): which + references the ``sum`` estimate uses. Ignored by the block modes. """ columns: ColumnSchema @@ -280,6 +293,7 @@ class ToggleSpecialistMethod: segment_imbalance_warning: float = 0.1 reference_mode: str = "sum" reference_block: pd.Timedelta | None = None + reference_choice: str = "all" def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" @@ -287,6 +301,9 @@ def __post_init__(self) -> None: if self.reference_mode not in REFERENCE_MODES: msg = f"{self.name}: reference_mode {self.reference_mode!r} is not one of {REFERENCE_MODES}" raise ValueError(msg) + if self.reference_choice not in REFERENCE_CHOICES: + msg = f"{self.name}: reference_choice {self.reference_choice!r} is not one of {REFERENCE_CHOICES}" + raise ValueError(msg) validate_conditions(self.conditions, supported=_SUPPORTED_CONDITIONS, method_name=self.name) if "power" in self.conditions and self.rated_power_kw is None: msg = ( @@ -331,12 +348,17 @@ def estimate(self, mi: MethodInput) -> MethodOutput: test, ) modes = ("block_mean",) - components = { - mode: self._component( - mi, mode=mode, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows - ) - for mode in modes - } + components: dict[str, _Estimate] = {} + subsets: pd.DataFrame | None = None + for mode in modes: + if mode == "sum" and self.reference_choice == "min_sigma": + components[mode], subsets = self._choose_references( + mi, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + ) + else: + components[mode] = self._component( + mi, mode=mode, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + ) est = ( _combine(components[_COMBINED_COMPONENTS[0]], components[_COMBINED_COMPONENTS[1]]) if modes == _COMBINED_COMPONENTS @@ -354,7 +376,17 @@ def estimate(self, mi: MethodInput) -> MethodOutput: timebase=timebase, active_power_col=self.columns.active_power, ) - self._write_outputs(mi, wide=wide, stats=stats, est=est, components=components, timebase=timebase, rows=rows) + self._write_outputs( + mi, + wide=wide, + stats=stats, + est=est, + components=components, + timebase=timebase, + rows=rows, + subsets=subsets, + n_refs_available=len(mi.context.references_among(wide_all.columns)), + ) return MethodOutput( p50_overall=float(est.uplift), p50_by_condition=est.per_bin, @@ -374,11 +406,17 @@ def _component( timebase: pd.Timedelta, block: pd.Timedelta | None, rows: ToggleRowSets, + refs: list[str] | None = None, ) -> _Estimate: - """Run one reference mode end to end: row selection, reference, uplift, bins and bootstrap.""" - # block_mean compares the test turbine against its own block, so references play no part, - # not even in the row filters: a reference outage must not cost a zero-reference run rows. - refs = [] if mode == "block_mean" else mi.context.references_among(wide_all.columns) + """Run one reference mode end to end: row selection, reference, uplift, bins and bootstrap. + + ``refs`` names the references to use; ``None`` means every available one (none for + ``block_mean``, which compares the test turbine against its own block, so references play + no part, not even in the row filters: a reference outage must not cost a zero-reference run + rows). + """ + if refs is None: + refs = [] if mode == "block_mean" else mi.context.references_among(wide_all.columns) if not refs and mode != "block_mean": msg = ( f"no reference turbines available for test_wtg {test!r}: scada_df contains only " @@ -468,6 +506,60 @@ def _component( diagnostics=diagnostics, ) + def _choose_references( + self, + mi: MethodInput, + *, + wide_all: pd.DataFrame, + test: str, + timebase: pd.Timedelta, + block: pd.Timedelta | None, + rows: ToggleRowSets, + ) -> tuple[_Estimate, pd.DataFrame | None]: + """Run the ``sum`` estimate on every non-empty reference subset; return the smallest-sigma one. + + Ties go to the larger subset, then to the earlier in enumeration (the context's reference + order). No finite sigma anywhere -> the full set. The second value lists every subset's + result with the winner marked; ``None`` when there were no references to choose among (the + full-set estimate then raises as it always has). + """ + refs_all = mi.context.references_among(wide_all.columns) + if not refs_all: + return self._component( + mi, mode="sum", wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + ), None + if len(refs_all) > _MAX_REFS_FOR_SUBSETS: + logger.warning( + "%s: reference_choice 'min_sigma' over %d references runs %d estimates", + self.name, + len(refs_all), + 2 ** len(refs_all) - 1, + ) + subsets = [list(c) for k in range(1, len(refs_all) + 1) for c in combinations(refs_all, k)] + estimates = [ + self._component( + mi, mode="sum", wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows, refs=refs + ) + for refs in subsets + ] + + def _rank(i: int) -> tuple[float, int, int]: + sigma = estimates[i].sigma_overall + return (sigma if np.isfinite(sigma) else np.inf, -len(subsets[i]), i) + + best = min(range(len(subsets)), key=_rank) + table = pd.DataFrame( + { + "refs": [";".join(refs) for refs in subsets], + "n_refs": [len(refs) for refs in subsets], + "n_used_timestamps": [int(est.used.sum()) for est in estimates], + "uplift_frc": [est.uplift for est in estimates], + "uplift_sigma_frc": [est.sigma_overall for est in estimates], + "chosen": [i == best for i in range(len(subsets))], + } + ) + return estimates[best], table + def _labeled_rows( self, mi: MethodInput, @@ -818,8 +910,10 @@ def _write_outputs( components: dict[str, _Estimate], timebase: pd.Timedelta, rows: ToggleRowSets, + subsets: pd.DataFrame | None = None, + n_refs_available: int | None = None, ) -> None: - """Write the data-stats CSV, the headline results CSV, the per-bin CSV and (optionally) the plots.""" + """Write the data-stats, results, per-bin, uncertainty, selection and reference-subsets CSVs and the plots.""" upgrade_start = toggle_upgrade_start(mi.upgrade_timing, wide.index) last_dt = wide.index.max() run_name = f"toggle_specialist_{mi.test_wtg}_{upgrade_start:%Y%m%d}_{last_dt:%Y%m%d}" @@ -841,7 +935,10 @@ def _write_outputs( "mode": "toggle", "n_turbines": wide.shape[1], "n_refs": len(est.refs), + "n_refs_available": len(est.refs) if n_refs_available is None else n_refs_available, + "refs_used": ";".join(est.refs), "reference_mode": self.reference_mode, + "reference_choice": self.reference_choice, "ratio_baseline": est.rho_base, "ratio_upgraded": est.rho_up, "uplift_frc": est.uplift, @@ -861,6 +958,8 @@ def _write_outputs( if est.per_bin is not None: est.per_bin.to_csv(run_dir / f"{run_name}_by_power_bin_{ts}.csv", index=False) est.diagnostics.to_csv(run_dir / f"{run_name}_uncertainty_{ts}.csv", index=False) + if subsets is not None: + subsets.to_csv(run_dir / f"{run_name}_reference_subsets_{ts}.csv", index=False) # Every component's row selection, so a blend's two selections can be told apart. pd.concat( [component.accounting.assign(component=mode) for mode, component in components.items()], ignore_index=True @@ -936,6 +1035,7 @@ def _write_shared_diagnostics( "reference_mode": self.reference_mode, "reference_block": None if self.reference_block is None else str(self.reference_block), "exclude_block_col": self.columns.exclude_block, + "reference_choice": self.reference_choice, } write_run_config(ctx, method_name=self.name, method_params=params) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index c8182005..e1dee00a 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1467,3 +1467,75 @@ def test_no_flags_change_nothing(self) -> None: without = _estimate(scada.drop(columns=_BLOCK_COL), schedule, reference_block=_BLOCK) assert with_role.p50_overall == pytest.approx(without.p50_overall) assert with_role.sigma_overall == pytest.approx(without.sigma_overall) + + +def _two_reference_case(noise_frac: float = 0.3, seed: int = 5) -> tuple[pd.DataFrame, ToggleSchedule]: + """``_block_case`` with a perfect tracker ``R1`` plus a noisy tracker ``R2``: the union is worse than R1 alone.""" + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5), seed=seed) + rng = np.random.default_rng(seed) + r2 = scada[scada[_TURBINE_COL] == "R1"].copy() + r2[_TURBINE_COL] = "R2" + r2[_POWER_COL] = r2[_POWER_COL].to_numpy() * (1.0 + noise_frac * rng.standard_normal(len(r2))) + return pd.concat([scada, r2]), schedule + + +class TestReferenceChoice: + """``reference_choice="min_sigma"``: the sum leg takes the reference subset with the smallest headline sigma.""" + + def test_the_default_keeps_every_reference(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + _estimate(scada, schedule, out_dir=tmp_path) + results = _read_only_csv(tmp_path, "results") + assert results.loc[0, "reference_choice"] == "all" + assert results.loc[0, "refs_used"] == "R1;R2" + assert results.loc[0, "n_refs"] == 2 + assert results.loc[0, "n_refs_available"] == 2 + + def test_min_sigma_picks_the_subset_with_the_smallest_sigma(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + out = _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path) + results = _read_only_csv(tmp_path, "results") + assert results.loc[0, "refs_used"] == "R1" + assert results.loc[0, "n_refs"] == 1 + assert results.loc[0, "n_refs_available"] == 2 + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + union = _estimate(scada, schedule) + assert out.sigma_overall < union.sigma_overall + + def test_every_subset_is_written_with_the_winner_marked(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path) + subsets = _read_only_csv(tmp_path, "reference_subsets") + assert sorted(subsets["refs"]) == ["R1", "R1;R2", "R2"] + assert subsets["chosen"].sum() == 1 + assert subsets.loc[subsets["chosen"], "refs"].item() == "R1" + assert subsets["uplift_sigma_frc"].notna().all() + assert (subsets["n_used_timestamps"] > 0).all() + + def test_combined_blends_the_chosen_subset(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + out = _estimate( + scada, + schedule, + reference_mode="combined", + reference_choice="min_sigma", + reference_block=_BLOCK, + out_dir=tmp_path, + ) + results = _read_only_csv(tmp_path, "results") + assert results.loc[0, "refs_used"] == "R1" + assert set(out.uncertainty_diagnostics["component"]) == {"sum", "block_mean", "combined"} + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + + def test_block_mean_ignores_the_choice(self) -> None: + scada, schedule = _two_reference_case() + chosen = _estimate( + scada, schedule, reference_mode="block_mean", reference_choice="min_sigma", reference_block=_BLOCK + ) + plain = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + assert chosen.p50_overall == pytest.approx(plain.p50_overall) + assert chosen.sigma_overall == pytest.approx(plain.sigma_overall) + + def test_an_unknown_choice_is_refused(self) -> None: + with pytest.raises(ValueError, match="reference_choice"): + ToggleSpecialistMethod(columns=_COLUMNS, reference_choice="best") From 68e554cef964af44e8546eaec1a8e80e30c96231 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Tue, 6 Oct 2026 09:31:30 +0100 Subject: [PATCH 11/18] Add decision_basis='permuted': judge the subset choice and blend weights on label-permuted bootstraps Ranking reference subsets by the realised uplift's bootstrap sigma, and weighting the combined blend by the legs' realised sigmas, both favour the alternative whose realised uplift happened to come out low, because the on/off contrast noise is skewed: a winner's curse that biases the estimate down and under-reports sigma, and does not shrink with more data. With decision_basis="permuted" each component also bootstraps the same rows with the on/off labels shuffled within each reference block (block on/off counts kept), and the min_sigma choice and the blend weights read that sigma instead. The reported uplift and sigma are always the actual ones. Subsets being ranked skip their actual bootstrap and the winner is re-run, so the cost is unchanged. combine_estimates gains a `weighting` argument (another bootstrap's sigmas and resamples to take the weights from); the blend's variance still uses the actual ones. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 60 ++++--- benchmarking/baselines/toggle_specialist.py | 156 +++++++++++++++--- .../baselines/test_toggle_specialist.py | 97 +++++++++++ 3 files changed, 273 insertions(+), 40 deletions(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 83d6c41e..294433ed 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -107,9 +107,12 @@ class CombinedEstimate: def combine_estimates( - a: tuple[float, float, npt.NDArray[np.float64]], b: tuple[float, float, npt.NDArray[np.float64]] + a: tuple[float, float, npt.NDArray[np.float64]], + b: tuple[float, float, npt.NDArray[np.float64]], + *, + weighting: tuple[float, float, npt.NDArray[np.float64], npt.NDArray[np.float64]] | None = None, ) -> CombinedEstimate: - """Blend two ``(estimate, sigma, resamples)`` of the same quantity by inverse variance. + """Blend two ``(estimate, sigma, resamples)`` of the same quantity by minimum-variance weights. A component is usable when both its estimate and its sigma are finite. With two usable components the weights are the minimum-variance ones for two correlated estimates, @@ -119,6 +122,12 @@ def combine_estimates( The blend's variance is ``wa² sa² + wb² sb² + 2 wa wb r sa sb``, which these weights keep at or below the better component's: a noisier leg correlated at ``r >= sb / sa`` gets no weight rather than widening the blend. One usable component is returned as is; none gives NaN throughout. + + ``weighting``, when given as ``(sigma_a, sigma_b, resamples_a, resamples_b)`` from another + bootstrap of the same two components (e.g. one with the on/off labels permuted), supplies the + weights instead, so they need not read the realised estimates' own spread; the blend's variance + and the reported correlation still come from ``a`` and ``b``. A ``weighting`` with a non-finite + sigma is ignored. """ est_a, sig_a, res_a = a est_b, sig_b, res_b = b @@ -132,6 +141,24 @@ def combine_estimates( if not (usable_a and usable_b): return CombinedEstimate(estimate=nan, sigma=nan, weight_a=nan, correlation=nan) + correlation = _paired_correlation(res_a, res_b) + if weighting is not None and math.isfinite(weighting[0]) and math.isfinite(weighting[1]): + sig_a_w, sig_b_w, res_a_w, res_b_w = weighting + weight_a = _min_variance_weight(sig_a_w, sig_b_w, _paired_correlation(res_a_w, res_b_w)) + else: + weight_a = _min_variance_weight(sig_a, sig_b, correlation) + weight_b = 1.0 - weight_a + variance = weight_a**2 * sig_a**2 + weight_b**2 * sig_b**2 + 2.0 * weight_a * weight_b * correlation * sig_a * sig_b + return CombinedEstimate( + estimate=weight_a * est_a + weight_b * est_b, + sigma=math.sqrt(max(variance, 0.0)), + weight_a=weight_a, + correlation=correlation, + ) + + +def _paired_correlation(res_a: npt.NDArray[np.float64], res_b: npt.NDArray[np.float64]) -> float: + """Pearson correlation of two components' draw-paired finite resamples, or the default.""" if len(res_a) and len(res_b) and len(res_a) != len(res_b): msg = ( f"the two components' resample arrays have different lengths ({len(res_a)} vs {len(res_b)}); " @@ -141,26 +168,19 @@ def combine_estimates( raise ValueError(msg) n_pairs = min(len(res_a), len(res_b)) paired = np.isfinite(res_a[:n_pairs]) & np.isfinite(res_b[:n_pairs]) - if paired.sum() >= _MIN_RESAMPLES_FOR_SPREAD: - r = float(np.corrcoef(res_a[:n_pairs][paired], res_b[:n_pairs][paired])[0, 1]) - correlation = r if math.isfinite(r) else _DEFAULT_COMPONENT_CORRELATION - else: - correlation = _DEFAULT_COMPONENT_CORRELATION + if paired.sum() < _MIN_RESAMPLES_FOR_SPREAD: + return _DEFAULT_COMPONENT_CORRELATION + r = float(np.corrcoef(res_a[:n_pairs][paired], res_b[:n_pairs][paired])[0, 1]) + return r if math.isfinite(r) else _DEFAULT_COMPONENT_CORRELATION + +def _min_variance_weight(sig_a: float, sig_b: float, correlation: float) -> float: + """Weight on ``a`` that minimises the variance of a blend of two estimates correlated at ``correlation``.""" if sig_a == 0.0 or sig_b == 0.0: - weight_a = 0.5 if sig_a == sig_b else float(sig_a == 0.0) - else: - cross = correlation * sig_a * sig_b - wa, wb = max(sig_b**2 - cross, 0.0), max(sig_a**2 - cross, 0.0) - weight_a = 0.5 if wa + wb == 0.0 else wa / (wa + wb) - weight_b = 1.0 - weight_a - variance = weight_a**2 * sig_a**2 + weight_b**2 * sig_b**2 + 2.0 * weight_a * weight_b * correlation * sig_a * sig_b - return CombinedEstimate( - estimate=weight_a * est_a + weight_b * est_b, - sigma=math.sqrt(max(variance, 0.0)), - weight_a=weight_a, - correlation=correlation, - ) + return 0.5 if sig_a == sig_b else float(sig_a == 0.0) + cross = correlation * sig_a * sig_b + wa, wb = max(sig_b**2 - cross, 0.0), max(sig_a**2 - cross, 0.0) + return 0.5 if wa + wb == 0.0 else wa / (wa + wb) def _nan_cells(names: list[str]) -> dict[str, CellUncertainty]: diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index ab814f22..bcac88f0 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -40,6 +40,15 @@ sigma (ties to the larger subset), so a reference that tracks badly, or is screened out for long stretches, loses to the subsets without it. Every subset's result is written to a CSV. +**Decision basis** (``decision_basis``) decides what those two data-driven decisions, the subset +choice and the blend weights, are judged on. ``"actual"`` ranks by the bootstrap sigma of the +realised uplift. Because the on/off contrast noise is skewed, the alternative whose realised sigma +is smallest tends to be the one whose realised uplift came out low, so ranking on it biases the +estimate down and under-reports sigma (a winner's curse). ``"permuted"`` instead judges each +alternative on a bootstrap of the same rows with the on/off labels shuffled within each reference +block: the same noise, blind to how it split between the states. The reported uplift and sigma are +always the actual ones; only the decisions move. + Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the uplift, from the uplift's own frozen row selection and bin assignment, and only when the uplift is @@ -110,6 +119,12 @@ _COMBINED_COMPONENTS: tuple[str, str] = ("sum", "block_mean") # ``reference_choice`` values; see the module docstring. REFERENCE_CHOICES: tuple[str, ...] = ("all", "min_sigma") +# ``decision_basis`` values: what the subset choice and the blend weights are judged on. ``"actual"`` +# uses the bootstrap of the realised uplift; ``"permuted"`` uses a bootstrap of the same rows with the +# on/off labels shuffled within each reference block, which measures the noise without seeing how it +# happened to split between the states, so a decision cannot favour the alternative whose realised +# uplift came out low (the winner's curse of ranking by realised sigma). +DECISION_BASES: tuple[str, ...] = ("actual", "permuted") # ``min_sigma`` runs 2**n - 1 estimates; warn, rather than refuse, past this many references. _MAX_REFS_FOR_SUBSETS = 6 @@ -143,6 +158,43 @@ class _Estimate: boot: BootstrapResult | None accounting: pd.DataFrame diagnostics: pd.DataFrame + # The label-permuted bootstrap the decisions read under ``decision_basis="permuted"``; None otherwise. + decision_boot: BootstrapResult | None = None + + @property + def decision_sigma(self) -> float: + """The headline sigma decisions are judged on: the permuted one when it exists, else the actual.""" + return _cell_sigma(self.decision_boot, _OVERALL) if self.decision_boot is not None else self.sigma_overall + + +def permute_labels_within_blocks( + *, + upgraded: npt.NDArray[np.bool_], + baseline: npt.NDArray[np.bool_], + used: npt.NDArray[np.bool_], + ids: npt.NDArray[np.int64] | None, + rng: np.random.Generator, +) -> tuple[npt.NDArray[np.bool_], npt.NDArray[np.bool_]]: + """Shuffle the on/off labels among the used campaign rows of each block; unused rows keep theirs. + + Every block keeps its on and off counts, so the permuted campaign has the same structure as the + real one and differs only in which rows are called on. With ``ids`` None the labels are shuffled + over the whole campaign. + """ + up_p = upgraded.copy() + base_p = baseline.copy() + campaign_used = np.flatnonzero(used & (upgraded | baseline)) + if ids is None: + groups = [campaign_used] + else: + groups = [campaign_used[ids[campaign_used] == b] for b in np.unique(ids[campaign_used])] + for rows in groups: + if len(rows) < 2: # noqa: PLR2004 - nothing to shuffle + continue + order = rng.permutation(rows) + up_p[rows] = upgraded[order] + base_p[rows] = baseline[order] + return up_p, base_p class PairedRows(NamedTuple): @@ -277,6 +329,10 @@ class ToggleSpecialistMethod: positive whole multiple of the timebase; defaults to ``pairing_max_gap`` (one cycle). :param reference_choice: one of :data:`REFERENCE_CHOICES` (see the module docstring): which references the ``sum`` estimate uses. Ignored by the block modes. + :param decision_basis: one of :data:`DECISION_BASES`: what the ``min_sigma`` choice and the + ``combined`` blend weights are judged on. ``"permuted"`` judges them on a bootstrap of the + same rows with the on/off labels shuffled within each reference block, so neither decision + can chase the realised uplift; the reported uplift and sigma are always the actual ones. """ columns: ColumnSchema @@ -294,6 +350,7 @@ class ToggleSpecialistMethod: reference_mode: str = "sum" reference_block: pd.Timedelta | None = None reference_choice: str = "all" + decision_basis: str = "actual" def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" @@ -304,6 +361,9 @@ def __post_init__(self) -> None: if self.reference_choice not in REFERENCE_CHOICES: msg = f"{self.name}: reference_choice {self.reference_choice!r} is not one of {REFERENCE_CHOICES}" raise ValueError(msg) + if self.decision_basis not in DECISION_BASES: + msg = f"{self.name}: decision_basis {self.decision_basis!r} is not one of {DECISION_BASES}" + raise ValueError(msg) validate_conditions(self.conditions, supported=_SUPPORTED_CONDITIONS, method_name=self.name) if "power" in self.conditions and self.rated_power_kw is None: msg = ( @@ -360,7 +420,9 @@ def estimate(self, mi: MethodInput) -> MethodOutput: mi, mode=mode, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows ) est = ( - _combine(components[_COMBINED_COMPONENTS[0]], components[_COMBINED_COMPONENTS[1]]) + _combine( + components[_COMBINED_COMPONENTS[0]], components[_COMBINED_COMPONENTS[1]], basis=self.decision_basis + ) if modes == _COMBINED_COMPONENTS else components[modes[0]] ) @@ -407,13 +469,15 @@ def _component( block: pd.Timedelta | None, rows: ToggleRowSets, refs: list[str] | None = None, + with_actual: bool = True, ) -> _Estimate: """Run one reference mode end to end: row selection, reference, uplift, bins and bootstrap. ``refs`` names the references to use; ``None`` means every available one (none for ``block_mean``, which compares the test turbine against its own block, so references play no part, not even in the row filters: a reference outage must not cost a zero-reference run - rows). + rows). ``with_actual=False`` skips the actual bootstrap (a subset that is only being ranked + on its permuted sigma); the label-permuted bootstrap runs whenever ``decision_basis`` asks. """ if refs is None: refs = [] if mode == "block_mean" else mi.context.references_among(wide_all.columns) @@ -465,20 +529,31 @@ def _component( # Uncertainty runs strictly after the uplift, off the same frozen row selection and bin # assignment, and only when there is a finite uplift to qualify. membership = self._cell_membership(rho_label=rho_label, ref_total=ref_total, used=used) - boot = ( - self._bootstrap( + + def _run_bootstrap(up: npt.NDArray[np.bool_], base: npt.NDArray[np.bool_]) -> BootstrapResult: + return self._bootstrap( index=wide.index, test_pw=test_pw, ref_total=ref_total, used=used, - upgraded=rows.upgraded, - baseline=baseline, + upgraded=up, + baseline=base, membership=membership, timebase=timebase, ) - if np.isfinite(uplift) - else None - ) + + boot = _run_bootstrap(rows.upgraded, baseline) if with_actual and np.isfinite(uplift) else None + decision_boot = None + if self.decision_basis == "permuted" and np.isfinite(uplift): + # Same rows, span, blocks, seed and cells as the actual bootstrap; only the labels differ. + up_p, base_p = permute_labels_within_blocks( + upgraded=rows.upgraded, + baseline=baseline, + used=used, + ids=ids, + rng=np.random.default_rng(self.bootstrap_seed), + ) + decision_boot = _run_bootstrap(up_p, base_p) if per_bin is not None: per_bin["sigma_uplift"] = [_cell_sigma(boot, str(b)) for b in per_bin["condition_bin"]] diagnostics = _uncertainty_diagnostics( @@ -504,6 +579,7 @@ def _component( boot=boot, accounting=accounting, diagnostics=diagnostics, + decision_boot=decision_boot, ) def _choose_references( @@ -518,10 +594,12 @@ def _choose_references( ) -> tuple[_Estimate, pd.DataFrame | None]: """Run the ``sum`` estimate on every non-empty reference subset; return the smallest-sigma one. - Ties go to the larger subset, then to the earlier in enumeration (the context's reference - order). No finite sigma anywhere -> the full set. The second value lists every subset's - result with the winner marked; ``None`` when there were no references to choose among (the - full-set estimate then raises as it always has). + The sigma ranked is the ``decision_sigma``: the actual headline sigma, or under + ``decision_basis="permuted"`` the label-permuted one (the subsets then skip their actual + bootstrap and the winner is re-run in full). Ties go to the larger subset, then to the + earlier in enumeration (the context's reference order). No finite sigma anywhere -> the full + set. The second value lists every subset's result with the winner marked; ``None`` when + there were no references to choose among (the full-set estimate then raises as it always has). """ refs_all = mi.context.references_among(wide_all.columns) if not refs_all: @@ -536,18 +614,38 @@ def _choose_references( 2 ** len(refs_all) - 1, ) subsets = [list(c) for k in range(1, len(refs_all) + 1) for c in combinations(refs_all, k)] + permuted = self.decision_basis == "permuted" estimates = [ self._component( - mi, mode="sum", wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows, refs=refs + mi, + mode="sum", + wide_all=wide_all, + test=test, + timebase=timebase, + block=block, + rows=rows, + refs=refs, + with_actual=not permuted, ) for refs in subsets ] def _rank(i: int) -> tuple[float, int, int]: - sigma = estimates[i].sigma_overall + sigma = estimates[i].decision_sigma return (sigma if np.isfinite(sigma) else np.inf, -len(subsets[i]), i) best = min(range(len(subsets)), key=_rank) + if permuted: + estimates[best] = self._component( + mi, + mode="sum", + wide_all=wide_all, + test=test, + timebase=timebase, + block=block, + rows=rows, + refs=subsets[best], + ) table = pd.DataFrame( { "refs": [";".join(refs) for refs in subsets], @@ -555,6 +653,7 @@ def _rank(i: int) -> tuple[float, int, int]: "n_used_timestamps": [int(est.used.sum()) for est in estimates], "uplift_frc": [est.uplift for est in estimates], "uplift_sigma_frc": [est.sigma_overall for est in estimates], + "decision_sigma_frc": [est.decision_sigma for est in estimates], "chosen": [i == best for i in range(len(subsets))], } ) @@ -939,12 +1038,14 @@ def _write_outputs( "refs_used": ";".join(est.refs), "reference_mode": self.reference_mode, "reference_choice": self.reference_choice, + "decision_basis": self.decision_basis, "ratio_baseline": est.rho_base, "ratio_upgraded": est.rho_up, "uplift_frc": est.uplift, "uplift_sigma_frc": est.sigma_overall, **{f"uplift_frc_{mode}": component.uplift for mode, component in components.items()}, **{f"uplift_sigma_frc_{mode}": component.sigma_overall for mode, component in components.items()}, + **{f"decision_sigma_frc_{mode}": c.decision_sigma for mode, c in components.items()}, "block_hours": self.block_hours, "n_resamples": self.n_resamples, "n_used_timestamps_baseline": used_base, @@ -1063,8 +1164,11 @@ def _per_bin_counterfactual( return rho_row * ref_total -def _combine(a: _Estimate, b: _Estimate) -> _Estimate: - """Blend two component estimates by inverse variance, headline and per bin. +def _combine(a: _Estimate, b: _Estimate, *, basis: str = "actual") -> _Estimate: + """Blend two component estimates by minimum-variance weights, headline and per bin. + + With ``basis="permuted"`` the weights (and the correlation they need) come from the two legs' + label-permuted bootstraps, cell by cell; the blend's sigma always comes from the actual ones. The blend's used rows are the union of the components' (a row contributed to at least one estimate); its bin label is ``a``'s where ``a`` used the row and ``b``'s otherwise, so the two @@ -1079,11 +1183,21 @@ def _combine(a: _Estimate, b: _Estimate) -> _Estimate: msg = f"component bootstraps drew different block counts ({a.boot.n_blocks} vs {b.boot.n_blocks})" raise ValueError(msg) - def _resamples(est: _Estimate, cell: str) -> npt.NDArray[np.float64]: - return est.boot.resamples.get(cell, np.array([])) if est.boot is not None else np.array([]) + def _resamples(boot: BootstrapResult | None, cell: str) -> npt.NDArray[np.float64]: + return boot.resamples.get(cell, np.array([])) if boot is not None else np.array([]) def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> CombinedEstimate: - return combine_estimates((est_a, sig_a, _resamples(a, cell)), (est_b, sig_b, _resamples(b, cell))) + weighting = None + if basis == "permuted": + weighting = ( + _cell_sigma(a.decision_boot, cell), + _cell_sigma(b.decision_boot, cell), + _resamples(a.decision_boot, cell), + _resamples(b.decision_boot, cell), + ) + return combine_estimates( + (est_a, sig_a, _resamples(a.boot, cell)), (est_b, sig_b, _resamples(b.boot, cell)), weighting=weighting + ) overall = _cell(_OVERALL, a.uplift, a.sigma_overall, b.uplift, b.sigma_overall) blended_cells = {_OVERALL: overall} @@ -1120,6 +1234,8 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> "sigma": blend.sigma, f"weight_{a.mode}": blend.weight_a, "correlation": blend.correlation, + f"decision_sigma_{a.mode}": _cell_sigma(a.decision_boot, cell), + f"decision_sigma_{b.mode}": _cell_sigma(b.decision_boot, cell), } for cell, blend in blended_cells.items() ] diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index e1dee00a..6ee9ce25 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1539,3 +1539,100 @@ def test_block_mean_ignores_the_choice(self) -> None: def test_an_unknown_choice_is_refused(self) -> None: with pytest.raises(ValueError, match="reference_choice"): ToggleSpecialistMethod(columns=_COLUMNS, reference_choice="best") + + +class TestDecisionBasis: + """``decision_basis="permuted"``: the subset choice and the blend weights come from label-permuted bootstraps.""" + + def test_an_unknown_basis_is_refused(self) -> None: + with pytest.raises(ValueError, match="decision_basis"): + ToggleSpecialistMethod(columns=_COLUMNS, decision_basis="lucky") + + def test_actual_is_the_default(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + _estimate(scada, schedule, out_dir=tmp_path) + results = _read_only_csv(tmp_path, "results") + assert results.loc[0, "decision_basis"] == "actual" + + def test_permuted_labels_keep_every_blocks_on_off_counts(self) -> None: + rng = np.random.default_rng(0) + n = 40 + upgraded = np.arange(n) % 4 < 2 # two on, two off per block of four + baseline = ~upgraded + used = np.ones(n, dtype=bool) + used[[3, 10, 11]] = False + ids = np.arange(n) // 4 + up_p, base_p = toggle_specialist.permute_labels_within_blocks( + upgraded=upgraded, baseline=baseline, used=used, ids=ids, rng=rng + ) + assert not np.array_equal(up_p, upgraded) + assert np.array_equal(up_p & used, ~base_p & used) # every used row is exactly one state + assert np.array_equal(up_p[~used], upgraded[~used]) # unused rows untouched + for block in np.unique(ids): + rows = (ids == block) & used + assert (up_p & rows).sum() == (upgraded & rows).sum() + assert (base_p & rows).sum() == (baseline & rows).sum() + + def test_permuted_labels_without_blocks_are_shuffled_globally(self) -> None: + rng = np.random.default_rng(1) + upgraded = np.arange(12) % 2 == 0 + up_p, base_p = toggle_specialist.permute_labels_within_blocks( + upgraded=upgraded, baseline=~upgraded, used=np.ones(12, dtype=bool), ids=None, rng=rng + ) + assert up_p.sum() == upgraded.sum() + assert np.array_equal(up_p, ~base_p) + + def test_min_sigma_ranks_the_subsets_by_the_permuted_sigma(self, tmp_path: Path) -> None: + scada, schedule = _two_reference_case() + out = _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted", out_dir=tmp_path) + subsets = _read_only_csv(tmp_path, "reference_subsets") + assert subsets["decision_sigma_frc"].notna().all() + chosen = subsets.loc[subsets["chosen"], "refs"].item() + assert chosen == subsets.loc[subsets["decision_sigma_frc"].idxmin(), "refs"] + assert chosen == "R1" # the perfect tracker is the least noisy on any basis + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + results = _read_only_csv(tmp_path, "results") + assert results.loc[0, "decision_basis"] == "permuted" + assert results.loc[0, "refs_used"] == "R1" + + def test_the_reported_sigma_is_the_actual_one_of_the_chosen_subset(self) -> None: + scada, schedule = _two_reference_case() + chosen = _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted") + only_r1 = _estimate(scada[scada[_TURBINE_COL] != "R2"], schedule) + assert chosen.p50_overall == pytest.approx(only_r1.p50_overall) + assert chosen.sigma_overall == pytest.approx(only_r1.sigma_overall) + # The permuted sigma is a decision aid, never the answer: it is not the reported sigma. + assert chosen.sigma_overall > 0 + + def test_the_permuted_sigma_measures_the_same_noise_as_the_actual_one(self, tmp_path: Path) -> None: + """For a noisy reference the two sigmas agree; they differ only in whether the on/off split is the real one.""" + scada, schedule = _two_reference_case() + _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path / "actual") + _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted", out_dir=tmp_path / "perm") + actual = _read_only_csv(tmp_path / "actual", "reference_subsets").set_index("refs")["uplift_sigma_frc"] + permuted = _read_only_csv(tmp_path / "perm", "reference_subsets").set_index("refs")["decision_sigma_frc"] + for refs in ("R2", "R1;R2"): # R1 alone is a perfect tracker: its actual sigma is exactly zero + assert permuted[refs] == pytest.approx(actual[refs], rel=0.3) + assert permuted["R1"] < permuted["R2"] + + def test_combined_weights_come_from_the_permuted_sigmas_but_the_sigma_is_actual(self) -> None: + scada, schedule = _noisy_toggle_case(n=600) + gap = pd.Timedelta(minutes=20) + out = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=gap, decision_basis="permuted") + diag = out.uncertainty_diagnostics + head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] + legs = { + m: diag[(diag["component"] == m) & (diag["condition_bin"] == "overall")].iloc[0] + for m in ("sum", "block_mean") + } + w = head["weight_sum"] + assert 0.0 <= w <= 1.0 + total = _estimate(scada, schedule, reference_mode="sum", pairing_max_gap=gap) + block = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=gap) + assert out.p50_overall == pytest.approx(w * total.p50_overall + (1 - w) * block.p50_overall) + sa, sb, r = legs["sum"]["sigma"], legs["block_mean"]["sigma"], head["correlation"] + assert sa == pytest.approx(total.sigma_overall) + variance = w**2 * sa**2 + (1 - w) ** 2 * sb**2 + 2 * w * (1 - w) * r * sa * sb + assert out.sigma_overall == pytest.approx(np.sqrt(variance)) + assert np.isfinite(head["decision_sigma_sum"]) + assert np.isfinite(head["decision_sigma_block_mean"]) From d1b742efe0008860b6ccb4f27f1792858d11a72c Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 10:21:43 +0100 Subject: [PATCH 12/18] ToggleSpecialist is one method: reference subset + block leg, blended, decided on permuted labels Remove the reference_mode, reference_choice and decision_basis options (and the block_scaled mode). The method is always the minimum-variance blend of the reference leg (the summed power of the reference subset with the smallest label-permuted sigma) and the block leg (the test turbine against its own one-cycle block mean), with the subset choice and the blend weights judged on label-permuted bootstraps and the reported uplift and sigma always the actual ones. With no reference turbine the estimate is the block leg alone. reference_block is now a required constructor argument: one toggle cycle, stated by the caller, independent of pairing_max_gap and never inferred. The blend's per-bin n_records counts the used upgraded rows by the merged bin label, so labeled_rows reproduces it; the data-stats CSV counts rows of the turbines the estimate used only. Co-Authored-By: Claude Fable 5.1 --- .../baselines/study_toggle_methods_compare.py | 1 + .../study_toggle_specialist_uncertainty.py | 1 + benchmarking/baselines/toggle_specialist.py | 347 +++++------ benchmarking/campaigns/methods.py | 4 +- .../baselines/test_toggle_specialist.py | 551 +++++++++--------- 5 files changed, 407 insertions(+), 497 deletions(-) diff --git a/benchmarking/baselines/study_toggle_methods_compare.py b/benchmarking/baselines/study_toggle_methods_compare.py index c4e3320f..f90fd6fe 100644 --- a/benchmarking/baselines/study_toggle_methods_compare.py +++ b/benchmarking/baselines/study_toggle_methods_compare.py @@ -202,6 +202,7 @@ def _build_methods(out_dir: Path, *, era5_hourly_df: pd.DataFrame) -> list: return [ ToggleSpecialistMethod( columns=HOT_COLUMNS, + reference_block=DEFAULT_TOGGLE_PERIOD, conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, out_dir=out_dir / "toggle_specialist_runs", diff --git a/benchmarking/baselines/study_toggle_specialist_uncertainty.py b/benchmarking/baselines/study_toggle_specialist_uncertainty.py index 44bfc536..b78a6ed1 100644 --- a/benchmarking/baselines/study_toggle_specialist_uncertainty.py +++ b/benchmarking/baselines/study_toggle_specialist_uncertainty.py @@ -129,6 +129,7 @@ def build_methods(block_hours_grid: list[float], out_dir: Path | None = None) -> return [ ToggleSpecialistMethod( columns=HOT_COLUMNS, + reference_block=DEFAULT_TOGGLE_PERIOD, name=f"toggle_specialist_bl{block_hours:g}", conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index bcac88f0..9b151b11 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -20,34 +20,27 @@ toggle cycle), so both states lose the same conditions — a symmetric alternative to ``exclude_row`` for a selection that would otherwise thin one state's rows in particular conditions. -**Reference modes** (``reference_mode``) decide what the test power is compared against. ``"sum"`` -is the summed power of the references. The two *block* modes tile the campaign into fixed -wall-clock blocks of ``reference_block`` (one toggle cycle by default) and give every used row of a -block the same reference value, so the on/off ratio cancels the block's wind level exactly: -``"block_mean"`` uses the block's mean test power and needs **no references at all**; -``"block_scaled"`` keeps the summed references but rescales them per block so each block's reference -total equals its test total, leaving the references only the within-block tracking. A block whose -used rows lack either state is dropped (the ``block`` selection stage). ``"combined"`` runs the -``sum`` and ``block_mean`` estimates, each on its own row selection, and blends them by inverse -variance, headline and per bin; the two bootstraps share their block draws, so the blend's sigma -includes the components' correlation (:func:`benchmarking.baselines.block_bootstrap.combine_estimates`). -Their errors come from different places (reference mismatch versus within-cycle drift), which is -what the blend exploits. - -**Reference choice** (``reference_choice``) decides which references the ``sum`` estimate (alone or -as the blend's reference leg) uses. ``"all"`` is every available reference; ``"min_sigma"`` runs the -``sum`` estimate on every non-empty subset of them and keeps the subset with the smallest headline -sigma (ties to the larger subset), so a reference that tracks badly, or is screened out for long -stretches, loses to the subsets without it. Every subset's result is written to a CSV. - -**Decision basis** (``decision_basis``) decides what those two data-driven decisions, the subset -choice and the blend weights, are judged on. ``"actual"`` ranks by the bootstrap sigma of the -realised uplift. Because the on/off contrast noise is skewed, the alternative whose realised sigma -is smallest tends to be the one whose realised uplift came out low, so ranking on it biases the -estimate down and under-reports sigma (a winner's curse). ``"permuted"`` instead judges each -alternative on a bootstrap of the same rows with the on/off labels shuffled within each reference -block: the same noise, blind to how it split between the states. The reported uplift and sigma are -always the actual ones; only the decisions move. +**Two legs, blended.** The test power is compared against two references, each estimated on +its own row selection, and the two estimates are blended by minimum-variance weights, headline and +per bin. The *reference leg* (``sum``) compares against the summed power of a **reference subset**: +the ``sum`` estimate runs on every non-empty subset of the available references and keeps the +subset whose noise is smallest (ties to the larger subset), so a reference that tracks badly, or is +screened out for long stretches, loses to the subsets without it; every subset's result is written +to a CSV. The *block leg* (``block_mean``) needs **no references at all**: the campaign is tiled +into fixed wall-clock blocks of ``reference_block`` (one toggle cycle) and every used row +of a block is compared against the block's mean test power, so the on/off ratio cancels the block's +wind level exactly; a block whose used rows lack either state is dropped (the ``block`` selection +stage). The two legs' errors come from different places (reference mismatch versus within-cycle +drift), which is what the blend exploits; their bootstraps share their block draws, so the blend's +sigma includes the legs' correlation (:func:`benchmarking.baselines.block_bootstrap.combine_estimates`). +With no reference turbine at all the estimate is the block leg alone. + +**Decisions on permuted labels.** The two data-driven decisions, the subset choice and the blend +weights, are judged on a bootstrap of the same rows with the on/off labels shuffled within each +reference block: the same noise, blind to how it happened to split between the states. Ranking on +the realised sigma would favour the alternative whose realised uplift came out low (the on/off +contrast noise is skewed), biasing the estimate down and under-reporting sigma. The reported uplift +and sigma are always the actual ones; only the decisions read the permuted bootstrap. Every uplift — the headline and each power bin — comes with a non-optional 1-sigma uncertainty from a circular block bootstrap (:mod:`benchmarking.baselines.block_bootstrap`). It is computed after the @@ -57,8 +50,9 @@ Each run writes a per-run folder ``toggle_specialist___/`` (v0-style naming) under ``out_dir`` (a temp dir by default), holding a per-segment data-stats CSV, -a headline results CSV, and -- when ``save_plots`` -- three diagnostic plots (a test-vs-reference -scatter, a per-segment daily-ratio timeseries, and a per-segment used-data-coverage timeseries). +a headline results CSV, the per-leg selection and reference-subset CSVs, and -- when +``save_plots`` -- three diagnostic plots (a test-vs-reference scatter, a per-segment daily-ratio +timeseries, and a per-segment used-data-coverage timeseries). The rich stats let a human confirm the right data was received and interpreted: the headline uplift is re-derivable from the stats CSV as ``rho = used_test_mwh / used_ref_total_mwh`` per segment. """ @@ -112,20 +106,12 @@ # treatment cannot move a row between bins. Binning by the test turbine's ws/TI would condition on # post-treatment signals, which this method exists not to do (see the module docstring). _SUPPORTED_CONDITIONS: tuple[str, ...] = ("power",) -# ``reference_mode`` values; see the module docstring. -REFERENCE_MODES: tuple[str, ...] = ("sum", "block_mean", "block_scaled", "combined") -# The two estimates ``"combined"`` blends, in blend order (the first is the one the row-selection -# accounting and the reference-side diagnostics report). -_COMBINED_COMPONENTS: tuple[str, str] = ("sum", "block_mean") -# ``reference_choice`` values; see the module docstring. -REFERENCE_CHOICES: tuple[str, ...] = ("all", "min_sigma") -# ``decision_basis`` values: what the subset choice and the blend weights are judged on. ``"actual"`` -# uses the bootstrap of the realised uplift; ``"permuted"`` uses a bootstrap of the same rows with the -# on/off labels shuffled within each reference block, which measures the noise without seeing how it -# happened to split between the states, so a decision cannot favour the alternative whose realised -# uplift came out low (the winner's curse of ranking by realised sigma). -DECISION_BASES: tuple[str, ...] = ("actual", "permuted") -# ``min_sigma`` runs 2**n - 1 estimates; warn, rather than refuse, past this many references. +# The two legs the estimate blends (see the module docstring), in blend order: the first is the one +# the row-selection accounting and the reference-side diagnostics report. +_REFERENCE_LEG = "sum" +_BLOCK_LEG = "block_mean" +_COMBINED = "combined" +# The subset choice runs 2**n - 1 estimates; warn, rather than refuse, past this many references. _MAX_REFS_FOR_SUBSETS = 6 logger = logging.getLogger(__name__) @@ -142,7 +128,7 @@ class _Selection(NamedTuple): @dataclass(frozen=True) class _Estimate: - """One reference mode's complete estimate: what the outputs, the labels and the blend read.""" + """One leg's (or the blend's) complete estimate: what the outputs, the labels and the blend read.""" mode: str refs: list[str] @@ -158,13 +144,13 @@ class _Estimate: boot: BootstrapResult | None accounting: pd.DataFrame diagnostics: pd.DataFrame - # The label-permuted bootstrap the decisions read under ``decision_basis="permuted"``; None otherwise. + # The label-permuted bootstrap the decisions read; None for the blend (which decides nothing further). decision_boot: BootstrapResult | None = None @property def decision_sigma(self) -> float: - """The headline sigma decisions are judged on: the permuted one when it exists, else the actual.""" - return _cell_sigma(self.decision_boot, _OVERALL) if self.decision_boot is not None else self.sigma_overall + """The headline sigma the decisions are judged on: that of the label-permuted bootstrap.""" + return _cell_sigma(self.decision_boot, _OVERALL) def permute_labels_within_blocks( @@ -323,19 +309,14 @@ class ToggleSpecialistMethod: a positive whole multiple of the timebase; ``None`` disables pairing. :param segment_imbalance_warning: warn when the two segments' kept fractions at any selection stage differ by more than this. - :param reference_mode: one of :data:`REFERENCE_MODES` (see the module docstring). ``"block_mean"`` - uses no references: they do not enter the row filters either, and none need exist. - :param reference_block: block length for the block modes and for ``columns.exclude_block``, a - positive whole multiple of the timebase; defaults to ``pairing_max_gap`` (one cycle). - :param reference_choice: one of :data:`REFERENCE_CHOICES` (see the module docstring): which - references the ``sum`` estimate uses. Ignored by the block modes. - :param decision_basis: one of :data:`DECISION_BASES`: what the ``min_sigma`` choice and the - ``combined`` blend weights are judged on. ``"permuted"`` judges them on a bootstrap of the - same rows with the on/off labels shuffled within each reference block, so neither decision - can chase the realised uplift; the reported uplift and sigma are always the actual ones. + :param reference_block: **required** block length for the block leg, the label permutation and + ``columns.exclude_block``: one toggle cycle (the campaign's own on + off duration), a + positive whole multiple of the timebase. It is the caller's to state, not inferred, and it + is independent of ``pairing_max_gap``. """ columns: ColumnSchema + reference_block: pd.Timedelta name: str = "toggle_specialist" out_dir: Path | None = None save_plots: bool = False @@ -347,23 +328,10 @@ class ToggleSpecialistMethod: bootstrap_seed: int = 0 pairing_max_gap: pd.Timedelta | None = None segment_imbalance_warning: float = 0.1 - reference_mode: str = "sum" - reference_block: pd.Timedelta | None = None - reference_choice: str = "all" - decision_basis: str = "actual" def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" self.columns.require_roles(("active_power", "availability")) - if self.reference_mode not in REFERENCE_MODES: - msg = f"{self.name}: reference_mode {self.reference_mode!r} is not one of {REFERENCE_MODES}" - raise ValueError(msg) - if self.reference_choice not in REFERENCE_CHOICES: - msg = f"{self.name}: reference_choice {self.reference_choice!r} is not one of {REFERENCE_CHOICES}" - raise ValueError(msg) - if self.decision_basis not in DECISION_BASES: - msg = f"{self.name}: decision_basis {self.decision_basis!r} is not one of {DECISION_BASES}" - raise ValueError(msg) validate_conditions(self.conditions, supported=_SUPPORTED_CONDITIONS, method_name=self.name) if "power" in self.conditions and self.rated_power_kw is None: msg = ( @@ -393,38 +361,32 @@ def estimate(self, mi: MethodInput) -> MethodOutput: test = mi.test_wtg timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) self._check_pairing_gap(timebase) - block = self._reference_block(timebase) rows = resolve_toggle(mi.upgrade_timing, wide_all.index) + block = self._reference_block(timebase) - modes: tuple[str, ...] = (self.reference_mode,) - if self.reference_mode == "combined": - modes = _COMBINED_COMPONENTS - if not mi.context.references_among(wide_all.columns): - # Nothing for the reference-based component to work with: the blend is its other - # half alone, which needs no references, rather than a failed estimate. - logger.warning( - "%s: no reference turbines available for test_wtg %r; 'combined' falls back to 'block_mean'", - self.name, - test, - ) - modes = ("block_mean",) components: dict[str, _Estimate] = {} subsets: pd.DataFrame | None = None - for mode in modes: - if mode == "sum" and self.reference_choice == "min_sigma": - components[mode], subsets = self._choose_references( - mi, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows - ) - else: - components[mode] = self._component( - mi, mode=mode, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows - ) - est = ( - _combine( - components[_COMBINED_COMPONENTS[0]], components[_COMBINED_COMPONENTS[1]], basis=self.decision_basis + n_refs_available = len(mi.context.references_among(wide_all.columns)) + if n_refs_available: + components[_REFERENCE_LEG], subsets = self._choose_references( + mi, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows ) - if modes == _COMBINED_COMPONENTS - else components[modes[0]] + else: + # Nothing for the reference leg to work with: the estimate is the block leg alone, which + # needs no references, rather than a failed estimate. + logger.warning( + "%s: no reference turbines available for test_wtg %r; the estimate is the %r leg alone", + self.name, + test, + _BLOCK_LEG, + ) + components[_BLOCK_LEG] = self._component( + mi, mode=_BLOCK_LEG, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + ) + est = ( + _combine(components[_REFERENCE_LEG], components[_BLOCK_LEG], upgraded=rows.upgraded, bins=self._bins()) + if _REFERENCE_LEG in components + else components[_BLOCK_LEG] ) ref_set = set(est.refs) @@ -447,7 +409,7 @@ def estimate(self, mi: MethodInput) -> MethodOutput: timebase=timebase, rows=rows, subsets=subsets, - n_refs_available=len(mi.context.references_among(wide_all.columns)), + n_refs_available=n_refs_available, ) return MethodOutput( p50_overall=float(est.uplift), @@ -471,22 +433,16 @@ def _component( refs: list[str] | None = None, with_actual: bool = True, ) -> _Estimate: - """Run one reference mode end to end: row selection, reference, uplift, bins and bootstrap. + """Run one leg end to end: row selection, reference, uplift, bins and the two bootstraps. - ``refs`` names the references to use; ``None`` means every available one (none for - ``block_mean``, which compares the test turbine against its own block, so references play - no part, not even in the row filters: a reference outage must not cost a zero-reference run - rows). ``with_actual=False`` skips the actual bootstrap (a subset that is only being ranked - on its permuted sigma); the label-permuted bootstrap runs whenever ``decision_basis`` asks. + ``refs`` names the references the ``sum`` leg uses (the block leg compares the test turbine + against its own block, so references play no part, not even in the row filters: a reference + outage must not cost it rows). ``with_actual=False`` skips the actual bootstrap (a subset + that is only being ranked on its permuted sigma); the label-permuted bootstrap always runs. """ - if refs is None: - refs = [] if mode == "block_mean" else mi.context.references_among(wide_all.columns) - if not refs and mode != "block_mean": - msg = ( - f"no reference turbines available for test_wtg {test!r}: scada_df contains only " - f"{list(wide_all.columns)}. The toggle specialist method needs at least one reference turbine " - f"in reference_mode {mode!r} (only 'block_mean' runs without references)." - ) + refs = [] if mode == _BLOCK_LEG else list(refs or []) + if mode == _REFERENCE_LEG and not refs: + msg = f"{self.name}: the {mode!r} leg needs at least one reference turbine" raise ValueError(msg) # Narrow to the mode's turbines and blank the cells the campaign says may not contribute, so # the estimate and every diagnostic see one consistent selection. @@ -502,7 +458,7 @@ def _component( else None ) selection = self._selection( - mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids, both_states=mode != "sum" + mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids, both_states=mode == _BLOCK_LEG ) used = selection.blocked.to_numpy() accounting = self._selection_accounting(selection, rows=rows) @@ -544,7 +500,7 @@ def _run_bootstrap(up: npt.NDArray[np.bool_], base: npt.NDArray[np.bool_]) -> Bo boot = _run_bootstrap(rows.upgraded, baseline) if with_actual and np.isfinite(uplift) else None decision_boot = None - if self.decision_basis == "permuted" and np.isfinite(uplift): + if np.isfinite(uplift): # Same rows, span, blocks, seed and cells as the actual bootstrap; only the labels differ. up_p, base_p = permute_labels_within_blocks( upgraded=rows.upgraded, @@ -592,40 +548,33 @@ def _choose_references( block: pd.Timedelta | None, rows: ToggleRowSets, ) -> tuple[_Estimate, pd.DataFrame | None]: - """Run the ``sum`` estimate on every non-empty reference subset; return the smallest-sigma one. - - The sigma ranked is the ``decision_sigma``: the actual headline sigma, or under - ``decision_basis="permuted"`` the label-permuted one (the subsets then skip their actual - bootstrap and the winner is re-run in full). Ties go to the larger subset, then to the - earlier in enumeration (the context's reference order). No finite sigma anywhere -> the full - set. The second value lists every subset's result with the winner marked; ``None`` when - there were no references to choose among (the full-set estimate then raises as it always has). + """Run the ``sum`` leg on every non-empty reference subset; return the smallest-sigma one. + + The sigma ranked is the ``decision_sigma`` of the label-permuted bootstrap; the subsets skip + their actual bootstrap and the winner is re-run in full. Ties go to the larger subset, then + to the earlier in enumeration (the context's reference order). No finite sigma anywhere -> + the full set. The second value lists every subset's result with the winner marked. """ refs_all = mi.context.references_among(wide_all.columns) - if not refs_all: - return self._component( - mi, mode="sum", wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows - ), None if len(refs_all) > _MAX_REFS_FOR_SUBSETS: logger.warning( - "%s: reference_choice 'min_sigma' over %d references runs %d estimates", + "%s: the reference-subset choice over %d references runs %d estimates", self.name, len(refs_all), 2 ** len(refs_all) - 1, ) subsets = [list(c) for k in range(1, len(refs_all) + 1) for c in combinations(refs_all, k)] - permuted = self.decision_basis == "permuted" estimates = [ self._component( mi, - mode="sum", + mode=_REFERENCE_LEG, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows, refs=refs, - with_actual=not permuted, + with_actual=False, ) for refs in subsets ] @@ -635,17 +584,16 @@ def _rank(i: int) -> tuple[float, int, int]: return (sigma if np.isfinite(sigma) else np.inf, -len(subsets[i]), i) best = min(range(len(subsets)), key=_rank) - if permuted: - estimates[best] = self._component( - mi, - mode="sum", - wide_all=wide_all, - test=test, - timebase=timebase, - block=block, - rows=rows, - refs=subsets[best], - ) + estimates[best] = self._component( + mi, + mode=_REFERENCE_LEG, + wide_all=wide_all, + test=test, + timebase=timebase, + block=block, + rows=rows, + refs=subsets[best], + ) table = pd.DataFrame( { "refs": [";".join(refs) for refs in subsets], @@ -785,6 +733,13 @@ def _conditional_frame( frame.insert(0, "condition", "power") return frame + def _bins(self) -> list[float] | None: + """Return the power bin edges when the method reports per bin; None otherwise.""" + if "power" not in self.conditions: + return None + assert self.rated_power_kw is not None # noqa: S101 - guaranteed by __post_init__ + return condition_bins("power", rated_power_kw=self.rated_power_kw) + def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: gap = self.pairing_max_gap if gap is None: @@ -796,21 +751,9 @@ def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: ) raise ValueError(msg) - def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta | None: - """Return the block length (reference_block, else pairing_max_gap); ``None`` when neither is set. - - The block modes (and ``combined``, for its block_mean component) cannot run without one and - raise; ``sum`` mode only needs it to honour ``columns.exclude_block``. - """ - block = self.reference_block if self.reference_block is not None else self.pairing_max_gap - if block is None: - if self.reference_mode == "sum": - return None - msg = ( - f"{self.name}: reference_mode {self.reference_mode!r} needs a block length: set reference_block " - f"or pairing_max_gap (one toggle cycle is the natural choice)." - ) - raise ValueError(msg) + def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta: + """Return ``reference_block`` after checking it tiles the timebase grid.""" + block = self.reference_block ratio = block / timebase if block <= pd.Timedelta(0) or ratio != round(ratio): msg = ( @@ -832,21 +775,17 @@ def _reference_total( ) -> npt.NDArray[np.float64]: """Return the per-row reference power for ``mode``; NaN on rows no surviving block covers. - Every used row of a block gets the same block-level factor, so within a block the on/off - ratio cancels the block's wind level whatever the on/off count mix (the block mean carries - the uplift at ``n_on / n`` strength in both denominators, a second-order effect). + The block leg gives every used row of a block the block's mean test power, so within a block + the on/off ratio cancels the block's wind level whatever the on/off count mix (the block mean + carries the uplift at ``n_on / n`` strength in both denominators, a second-order effect). """ - ref_sum = wide[refs].sum(axis=1).to_numpy(dtype=float) if refs else np.zeros(len(wide)) - if mode == "sum": - return ref_sum + if mode == _REFERENCE_LEG: + return wide[refs].sum(axis=1).to_numpy(dtype=float) if ids is None or not used.any(): return np.full(len(wide), np.nan) block_test = _block_sum(ids, test_pw, used) - if mode == "block_mean": - n_used = pd.Series(used.astype(float)).groupby(ids).sum() - return _per_row(ids, block_test / n_used.reindex(block_test.index)) - scale = block_test / _block_sum(ids, ref_sum, used) - return ref_sum * _per_row(ids, scale) + n_used = pd.Series(used.astype(float)).groupby(ids).sum() + return _per_row(ids, block_test / n_used.reindex(block_test.index)) def _selection( self, @@ -862,10 +801,9 @@ def _selection( ) -> _Selection: """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``. - ``ids`` are the reference-block ids (``None`` with no block length, or no campaign rows to - tile). The ``block`` stage drops every row of a block that carries a ``columns.exclude_block`` - flag on any of the turbines (all modes) or, when ``both_states`` (the block modes), lacks - either state. + ``ids`` are the reference-block ids (``None`` when there are no campaign rows to tile). The + ``block`` stage drops every row of a block that carries a ``columns.exclude_block`` flag on + any of the turbines (both legs) or, when ``both_states`` (the block leg), lacks either state. """ turbines = [test, *refs] filtered = self._filtered_mask(mi, wide=wide, test=test, refs=refs, timebase=timebase) @@ -886,7 +824,7 @@ def _selection( if ids is None: msg = ( f"{self.name}: columns.exclude_block {self.columns.exclude_block!r} carries flags but there is " - f"no block to remove: set reference_block or pairing_max_gap (one toggle cycle)." + f"no campaign row to tile into blocks." ) raise ValueError(msg) kept = kept & ~np.isin(ids, np.unique(ids[block_flag])) @@ -1036,9 +974,6 @@ def _write_outputs( "n_refs": len(est.refs), "n_refs_available": len(est.refs) if n_refs_available is None else n_refs_available, "refs_used": ";".join(est.refs), - "reference_mode": self.reference_mode, - "reference_choice": self.reference_choice, - "decision_basis": self.decision_basis, "ratio_baseline": est.rho_base, "ratio_upgraded": est.rho_up, "uplift_frc": est.uplift, @@ -1061,7 +996,7 @@ def _write_outputs( est.diagnostics.to_csv(run_dir / f"{run_name}_uncertainty_{ts}.csv", index=False) if subsets is not None: subsets.to_csv(run_dir / f"{run_name}_reference_subsets_{ts}.csv", index=False) - # Every component's row selection, so a blend's two selections can be told apart. + # Every leg's row selection, so the blend's two selections can be told apart. pd.concat( [component.accounting.assign(component=mode) for mode, component in components.items()], ignore_index=True ).to_csv(run_dir / f"{run_name}_selection_{ts}.csv", index=False) @@ -1133,10 +1068,8 @@ def _write_shared_diagnostics( "active_power_col": self.columns.active_power, "availability_col": self.columns.availability, "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), - "reference_mode": self.reference_mode, - "reference_block": None if self.reference_block is None else str(self.reference_block), + "reference_block": str(self.reference_block), "exclude_block_col": self.columns.exclude_block, - "reference_choice": self.reference_choice, } write_run_config(ctx, method_name=self.name, method_params=params) @@ -1164,18 +1097,19 @@ def _per_bin_counterfactual( return rho_row * ref_total -def _combine(a: _Estimate, b: _Estimate, *, basis: str = "actual") -> _Estimate: - """Blend two component estimates by minimum-variance weights, headline and per bin. +def _combine(a: _Estimate, b: _Estimate, *, upgraded: npt.NDArray[np.bool_], bins: list[float] | None) -> _Estimate: + """Blend the two legs' estimates by minimum-variance weights, headline and per bin. - With ``basis="permuted"`` the weights (and the correlation they need) come from the two legs' - label-permuted bootstraps, cell by cell; the blend's sigma always comes from the actual ones. + The weights (and the correlation they need) come from the two legs' label-permuted bootstraps, + cell by cell; the blend's sigma comes from the actual ones. The blend's used rows are the union of the components' (a row contributed to at least one estimate); its bin label is ``a``'s where ``a`` used the row and ``b``'s otherwise, so the two - components' bins, which share edges, are merged by name. Row-selection accounting, ``rho`` and - the reference-side diagnostics are ``a``'s (the reference-based component); ``b``'s selection - is written to the selection CSV. The diagnostics carry both components' rows and a ``combined`` - row per cell with the blend's sigma, the weight on ``a`` and the correlation it used. + components' bins, which share edges, are merged by name, and each bin's ``n_records`` counts + the used ``upgraded`` rows that merged label puts in it (what ``labeled_rows`` reproduces). + Row-selection accounting, ``rho`` and the reference-side diagnostics are ``a``'s (the reference + leg); ``b``'s selection is written to the selection CSV. The diagnostics carry both legs' rows and + a ``combined`` row per cell with the blend's sigma, the weight on ``a`` and the correlation it used. """ if a.boot is not None and b.boot is not None and a.boot.n_blocks != b.boot.n_blocks: # Both bootstraps come from the same method instance (same span, timebase, block length, @@ -1187,14 +1121,12 @@ def _resamples(boot: BootstrapResult | None, cell: str) -> npt.NDArray[np.float6 return boot.resamples.get(cell, np.array([])) if boot is not None else np.array([]) def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> CombinedEstimate: - weighting = None - if basis == "permuted": - weighting = ( - _cell_sigma(a.decision_boot, cell), - _cell_sigma(b.decision_boot, cell), - _resamples(a.decision_boot, cell), - _resamples(b.decision_boot, cell), - ) + weighting = ( + _cell_sigma(a.decision_boot, cell), + _cell_sigma(b.decision_boot, cell), + _resamples(a.decision_boot, cell), + _resamples(b.decision_boot, cell), + ) return combine_estimates( (est_a, sig_a, _resamples(a.boot, cell)), (est_b, sig_b, _resamples(b.boot, cell)), weighting=weighting ) @@ -1202,10 +1134,13 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> overall = _cell(_OVERALL, a.uplift, a.sigma_overall, b.uplift, b.sigma_overall) blended_cells = {_OVERALL: overall} + used = a.used | b.used + label = np.where(a.used, a.label, b.label) per_bin = None - if a.per_bin is not None and b.per_bin is not None: + if a.per_bin is not None and b.per_bin is not None and bins is not None: fa = a.per_bin.set_index(a.per_bin["condition_bin"].astype(str)) fb = b.per_bin.set_index(b.per_bin["condition_bin"].astype(str)) + upgraded_bin = pd.Series(pd.cut(label[used & upgraded], bins=bins).astype(str)) records = [] for cell in fa.index: ra, rb = fa.loc[cell], fb.loc[cell] @@ -1217,7 +1152,7 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> "condition": ra["condition"], "condition_bin": ra["condition_bin"], "p50_uplift": blend.estimate, - "n_records": int(max(ra["n_records"], rb["n_records"])), + "n_records": int((upgraded_bin == cell).sum()), "sum_actual": take["sum_actual"], "sum_counterfactual": take["sum_counterfactual"], "sigma_uplift": blend.sigma, @@ -1228,7 +1163,7 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> combined_rows = pd.DataFrame( [ { - "component": "combined", + "component": _COMBINED, "condition": _OVERALL if cell == _OVERALL else "power", "condition_bin": cell, "sigma": blend.sigma, @@ -1243,12 +1178,12 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> diagnostics = pd.concat([a.diagnostics, b.diagnostics, combined_rows], ignore_index=True) return _Estimate( - mode="combined", + mode=_COMBINED, refs=a.refs, selection=a.selection, - used=a.used | b.used, + used=used, ref_total=a.ref_total, - label=np.where(a.used, a.label, b.label), + label=label, rho_base=a.rho_base, rho_up=a.rho_up, uplift=overall.estimate, @@ -1345,11 +1280,13 @@ def _segment_stats( row_rows = resolve_toggle(mi.upgrade_timing, mi.scada_df.index) row_power = mi.scada_df[active_power_col].to_numpy(dtype=float) + # Only the turbines the estimate used (the test and its chosen references) count as rows. + in_wide = mi.scada_df[mi.turbine_col].isin(wide.columns).to_numpy() ts_baseline = toggle_rows.campaign_baseline - row_baseline = row_rows.campaign_baseline + row_baseline = row_rows.campaign_baseline & in_wide ts_masks = {"all": np.ones(len(wide), dtype=bool), "baseline": ts_baseline, "upgraded": toggle_rows.upgraded} - row_masks = {"all": np.ones(len(mi.scada_df), dtype=bool), "baseline": row_baseline, "upgraded": row_rows.upgraded} + row_masks = {"all": in_wide, "baseline": row_baseline, "upgraded": row_rows.upgraded & in_wide} rows = [] for segment in _SEGMENTS: @@ -1459,7 +1396,7 @@ def _save_plots( toggle_rows = resolve_toggle(mi.upgrade_timing, wide.index) baseline_mask = toggle_rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) - ref_label = f"sum of reference {active_power_col}" if reference_mode == "sum" else f"{reference_mode} reference" + ref_label = f"{reference_mode} reference {active_power_col}" upgrade_start = toggle_upgrade_start(mi.upgrade_timing, wide.index) segments = ( ("baseline", used & baseline_mask, "C0"), diff --git a/benchmarking/campaigns/methods.py b/benchmarking/campaigns/methods.py index f708edca..4a1216e8 100644 --- a/benchmarking/campaigns/methods.py +++ b/benchmarking/campaigns/methods.py @@ -8,7 +8,7 @@ from benchmarking.baselines.power_model import CURATED_ERA5_EXCLUDE, TUNED_MODEL_PARAMS, PowerModelMethod from benchmarking.baselines.toggle_specialist import ToggleSpecialistMethod from benchmarking.diagnostics.context import ERA5_UNLOCATED -from benchmarking.synthetic import HOT_COLUMNS +from benchmarking.synthetic import HOT_COLUMNS, ToggleSchedule if TYPE_CHECKING: from pathlib import Path @@ -52,9 +52,11 @@ def carried_forward_methods( """ methods: list[Method] = [NaiveRatioMethod(columns=HOT_COLUMNS, out_dir=out_dir / "naive_ratio", save_plots=True)] if spec.mode == "toggle": + assert isinstance(spec.upgrade_timing, ToggleSchedule) # noqa: S101 - what ``mode == "toggle"`` means methods.append( ToggleSpecialistMethod( columns=HOT_COLUMNS, + reference_block=spec.upgrade_timing.period, out_dir=out_dir / "toggle_specialist", save_plots=True, conditions=("power",), diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 6ee9ce25..ed6929ce 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -43,6 +43,8 @@ # Larger than any test timebase's full period, so the (required) availability filter keeps # every row unless a test deliberately sets a lower value. _FULLY_AVAILABLE_SECS = 3600.0 +# One toggle cycle of the 20-minute ``ToggleSchedule`` most tests use (``reference_block`` is required). +_CYCLE = pd.Timedelta(minutes=20) # The method reads active_power + availability from the schema; the other required roles are # unused by the ratio, so name them with placeholders. _COLUMNS = ColumnSchema( @@ -119,7 +121,7 @@ def test_toggle_specialist_shares_no_wind_up_code() -> None: class TestRecovery: def test_toggle_recovers_known_uplift(self) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert isinstance(out, MethodOutput) @@ -171,9 +173,9 @@ def _varying_rho_case( def _per_bin(scada: pd.DataFrame, schedule: ToggleSchedule) -> pd.DataFrame: """Run the method with power conditioning on and return its populated per-bin rows.""" - out = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW).estimate( - MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + out = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None frame = out.p50_by_condition return frame[frame["n_records"] > 0] @@ -247,7 +249,7 @@ class TestConditionsConfiguration: def test_default_reports_no_conditions(self) -> None: # back-compat: existing callers get exactly today's behaviour and need no rating scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_by_condition is None @@ -256,17 +258,17 @@ def test_power_conditioning_does_not_move_the_headline(self) -> None: # the per-bin decomposition is additional reporting, never a change to the estimate scada, schedule, _ = _toggle_case(uplift=0.03) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - plain = ToggleSpecialistMethod(columns=_COLUMNS).estimate(mi) + plain = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate(mi) conditioned = ToggleSpecialistMethod( - columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(mi) assert conditioned.p50_overall == plain.p50_overall def test_frame_is_labelled_with_the_power_condition(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) - out = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW).estimate( - MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + out = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert set(out.p50_by_condition["condition"]) == {"power"} for col in ("condition_bin", "p50_uplift", "n_records", "sum_actual", "sum_counterfactual"): @@ -274,15 +276,19 @@ def test_frame_is_labelled_with_the_power_condition(self) -> None: def test_ws_condition_raises_citing_the_method_limit(self) -> None: with pytest.raises(ValueError, match="does not support"): - ToggleSpecialistMethod(columns=_COLUMNS, conditions=("ws",), rated_power_kw=_RATED_KW) + ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("ws",), rated_power_kw=_RATED_KW + ) def test_unknown_condition_raises(self) -> None: with pytest.raises(ValueError, match="unknown condition"): - ToggleSpecialistMethod(columns=_COLUMNS, conditions=("bogus",), rated_power_kw=_RATED_KW) + ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("bogus",), rated_power_kw=_RATED_KW + ) def test_power_without_a_rating_raises(self) -> None: with pytest.raises(ValueError, match="rated_power_kw"): - ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",)) + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",)) class TestPerBinSparseData: @@ -291,9 +297,9 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: # do about it, and an imputed prior would manufacture false confidence. Rating the turbine far # above the data's power range guarantees empty upper bins rather than hoping for them. scada, schedule = _varying_rho_case(uplift=0.05) - out = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=4000.0).estimate( - MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + out = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=4000.0 + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None empty = out.p50_by_condition[out.p50_by_condition["n_records"] == 0] assert not empty.empty @@ -301,9 +307,9 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: def test_every_bin_is_represented(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) - out = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW).estimate( - MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + out = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert len(out.p50_by_condition) == len(condition_bins("power", rated_power_kw=_RATED_KW)) - 1 @@ -312,13 +318,13 @@ class TestPerBinDiagnostics: def test_per_bin_csv_is_written(self, tmp_path: Path) -> None: scada, schedule = _varying_rho_case(uplift=0.05) ToggleSpecialistMethod( - columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert list(tmp_path.rglob("*_by_power_bin_*.csv")) def test_no_per_bin_csv_when_conditioning_is_off(self, tmp_path: Path) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - ToggleSpecialistMethod(columns=_COLUMNS, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert not list(tmp_path.rglob("*_by_power_bin_*.csv")) @@ -327,6 +333,7 @@ def test_per_bin_plot_written_with_save_plots(self, tmp_path: Path) -> None: scada, schedule = _varying_rho_case(uplift=0.05) ToggleSpecialistMethod( columns=_COLUMNS, + reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path, @@ -344,7 +351,7 @@ def test_prepost_timestamp_raises(self) -> None: treated = np.asarray(idx >= upgrade) scada = _recovery_scada(idx, treated=treated, uplift=0.05) with pytest.raises(ValueError, match="toggle"): - ToggleSpecialistMethod(columns=_COLUMNS).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=upgrade, turbine_col=_TURBINE_COL) ) @@ -361,13 +368,13 @@ def test_blank_availability_role_raises(self, blank: str) -> None: # A schema that leaves the availability role blank (empty or whitespace) would silently skip # downtime filtering, so construction must reject it. with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=replace(_COLUMNS, availability=blank)) + ToggleSpecialistMethod(columns=replace(_COLUMNS, availability=blank), reference_block=_CYCLE) def test_missing_availability_column_raises(self) -> None: scada, schedule, _ = _toggle_case() scada = scada.drop(columns=[_AVAIL_COL]) with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=_COLUMNS).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -380,7 +387,7 @@ def test_down_reference_timestamps_are_excluded(self) -> None: down = (corrupted[_TURBINE_COL] == "R1") & corrupted.index.isin([idx[3], idx[4]]) corrupted.loc[down, _POWER_COL] = 1e6 corrupted.loc[down, _AVAIL_COL] = 0.0 - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(0.05) @@ -390,7 +397,7 @@ class TestCompleteCase: def test_drops_timestamp_when_a_reference_is_nan(self) -> None: scada, schedule, _ = _toggle_case() idx = scada.index.unique().sort_values() - clean = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + clean = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -399,7 +406,7 @@ def test_drops_timestamp_when_a_reference_is_nan(self) -> None: corrupted = scada.copy() mask = (corrupted[_TURBINE_COL] == "R1") & (corrupted.index == idx[3]) corrupted.loc[mask, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(clean.p50_overall) @@ -410,12 +417,12 @@ def test_nan_at_unused_timestamp_does_not_change_estimate(self) -> None: # make idx[2] already unused (test NaN), then add a second NaN at the same timestamp base = scada.copy() base.loc[(base[_TURBINE_COL] == "T1") & (base.index == idx[2]), _POWER_COL] = np.nan - before = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + before = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=base, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) after_df = base.copy() after_df.loc[(after_df[_TURBINE_COL] == "R1") & (after_df.index == idx[2]), _POWER_COL] = np.nan - after = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + after = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=after_df, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert after.p50_overall == pytest.approx(before.p50_overall) @@ -430,7 +437,7 @@ def test_estimate_invariant_to_timebase(self) -> None: treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.04) results.append( - ToggleSpecialistMethod(columns=_COLUMNS) + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=schedule.period) .estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) .p50_overall ) @@ -440,6 +447,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 scada, schedule, _ = _toggle_case() method = ToggleSpecialistMethod( columns=_COLUMNS, + reference_block=pd.Timedelta(hours=1), out_dir=tmp_path, timebase=pd.Timedelta(minutes=30), ) @@ -454,7 +462,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 class TestActivePowerOnly: def test_extra_columns_ignored(self) -> None: scada, schedule, _ = _toggle_case() - plain = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + plain = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -462,21 +470,23 @@ def test_extra_columns_ignored(self) -> None: rng = np.random.default_rng(0) with_extra["ws"] = rng.normal(size=len(with_extra)) with_extra["rpm"] = rng.normal(size=len(with_extra)) - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=with_extra, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(plain.p50_overall) class TestErrors: - def test_no_reference_raises(self) -> None: + def test_no_reference_is_the_block_leg_alone(self, caplog: pytest.LogCaptureFixture) -> None: idx = _index(40) schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=idx[0]) scada = _scada({"T1": np.ones(len(idx))}, idx) - with pytest.raises(ValueError, match="reference"): - ToggleSpecialistMethod(columns=_COLUMNS).estimate( + with caplog.at_level("WARNING"): + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) + assert set(out.uncertainty_diagnostics["component"]) == {"block_mean"} + assert "no reference turbines" in caplog.text def test_no_used_baseline_returns_nan(self) -> None: scada, schedule, treated = _toggle_case() @@ -485,7 +495,7 @@ def test_no_used_baseline_returns_nan(self) -> None: off_ts = idx[~treated] is_baseline_test = (scada[_TURBINE_COL] == "T1") & scada.index.isin(off_ts) scada.loc[is_baseline_test, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert np.isnan(out.p50_overall) @@ -538,7 +548,9 @@ def _read_only_csv(folder: Path, kind: str) -> pd.DataFrame: class TestDiagnostics: def _run(self, tmp_path, *, save_plots: bool = False): # noqa: ANN001, ANN202 scada, schedule, _ = _toggle_case(uplift=0.06) - method = ToggleSpecialistMethod(columns=_COLUMNS, out_dir=tmp_path, save_plots=save_plots) + method = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=save_plots + ) out = method.estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -563,7 +575,8 @@ def test_results_csv_matches_estimate(self, tmp_path) -> None: # noqa: ANN001 results = _read_only_csv(tmp_path, "results").iloc[0] assert results["uplift_frc"] == pytest.approx(out.p50_overall) assert results["mode"] == "toggle" - assert results["n_refs"] == 2 + assert results["n_refs_available"] == 2 + assert results["n_refs"] == len(str(results["refs_used"]).split(";")) def test_stats_coverage_full_for_complete_data(self, tmp_path) -> None: # noqa: ANN001 self._run(tmp_path) @@ -596,7 +609,7 @@ def test_precampaign_dropped_from_baseline(self, tmp_path) -> None: # noqa: ANN schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=start) treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.05) - ToggleSpecialistMethod(columns=_COLUMNS, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) baseline = _read_only_csv(tmp_path, "data_stats").set_index("segment").loc["baseline"] @@ -617,6 +630,7 @@ def _noisy_toggle_case(n: int = 2016, *, uplift: float = 0.03, noise_frac: float def _estimate(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + kwargs.setdefault("reference_block", _CYCLE) return ToggleSpecialistMethod(columns=_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -644,7 +658,7 @@ def test_diagnostics_cover_the_headline_and_every_bin(self) -> None: out = _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW) diag = out.uncertainty_diagnostics assert diag is not None - assert (diag["condition"] == "overall").sum() == 1 + assert (diag["condition"] == "overall").sum() == diag["component"].nunique() assert set(diag.columns) >= { "condition", "condition_bin", @@ -654,38 +668,43 @@ def test_diagnostics_cover_the_headline_and_every_bin(self) -> None: "sigma_robust", "frac_resamples_finite", } - overall = diag[diag["condition"] == "overall"].iloc[0] + overall = diag[(diag["condition"] == "overall") & (diag["component"] == "sum")].iloc[0] assert overall["n_upgraded_records"] > 0 assert overall["n_baseline_records"] > 0 -class TestUncertaintyDoesNotChangeUplift: - def test_block_length_moves_the_bootstrap_and_nothing_else(self) -> None: - """The session's hard constraint, at the method's own seam. +class TestUncertaintyDoesNotChangeAnUplift: + """Every candidate's uplift is fixed before any bootstrap runs. - Asserted on the *bootstrap* component, not the reported sigma: the reported value is - ``max(bootstrap, fallback)``, and the fallback does not depend on block length — so on data - where it wins at both lengths it would mask the difference this test exists to check. - """ - scada, schedule = _noisy_toggle_case() - a = _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW, block_hours=6.0) - b = _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW, block_hours=96.0) - assert a.p50_overall == b.p50_overall - pd.testing.assert_series_equal( - a.p50_by_condition["p50_uplift"], - b.p50_by_condition["p50_uplift"], # type: ignore[index] - ) - boots = [ - out.uncertainty_diagnostics.set_index("condition_bin").loc["overall", "sigma_bootstrap"] # type: ignore[union-attr] - for out in (a, b) - ] - assert boots[0] != boots[1] + The bootstraps decide which reference subset is kept and how the legs are weighted, so the + headline may move with them; no subset's uplift and not the block leg's may. + """ - def test_uplift_matches_a_run_with_the_bootstrap_reduced_to_nothing(self) -> None: + @staticmethod + def _candidates(tmp_path: Path, **kwargs: object) -> tuple[pd.Series, float, pd.DataFrame]: scada, schedule = _noisy_toggle_case() - full = _estimate(scada, schedule, n_resamples=1000) - minimal = _estimate(scada, schedule, n_resamples=2) - assert full.p50_overall == minimal.p50_overall + out = _estimate(scada, schedule, out_dir=tmp_path, **kwargs) + subsets = _read_only_csv(tmp_path, "reference_subsets").set_index("refs")["uplift_frc"] + block = _read_only_csv(tmp_path, "results").loc[0, "uplift_frc_block_mean"] + diag = out.uncertainty_diagnostics.set_index(["component", "condition_bin"]) + return subsets, block, diag + + def test_block_length_moves_the_bootstrap_and_no_uplift(self, tmp_path: Path) -> None: + """Asserted on the *bootstrap* sigma, not the reported one: the reported value is + ``max(bootstrap, fallback)`` and the fallback does not depend on block length. + """ + subsets_a, block_a, da = self._candidates(tmp_path / "a", block_hours=6.0) + subsets_b, block_b, db = self._candidates(tmp_path / "b", block_hours=96.0) + pd.testing.assert_series_equal(subsets_a, subsets_b) + assert block_a == block_b + for leg in ("sum", "block_mean"): + assert da.loc[(leg, "overall"), "sigma_bootstrap"] != db.loc[(leg, "overall"), "sigma_bootstrap"] + + def test_no_uplift_moves_when_the_bootstrap_is_reduced_to_nothing(self, tmp_path: Path) -> None: + subsets_full, block_full, _ = self._candidates(tmp_path / "full", n_resamples=1000) + subsets_min, block_min, _ = self._candidates(tmp_path / "minimal", n_resamples=2) + pd.testing.assert_series_equal(subsets_full, subsets_min) + assert block_full == block_min class TestUncertaintyRunsOnlyWhenThereIsAnUpliftToQualify: @@ -694,7 +713,7 @@ def test_a_nan_uplift_reports_a_nan_sigma(self) -> None: idx = _index(40) # every row treated -> no off rows -> rho_base is NaN scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput( scada_df=scada, test_wtg="T1", @@ -710,7 +729,9 @@ def test_the_bootstrap_is_not_run_when_the_uplift_is_nan(self) -> None: """Diagnostics still report the counts: they are what explains why there was no answer.""" idx = _index(40) scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) - out = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW).estimate( + out = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + ).estimate( MethodInput( scada_df=scada, test_wtg="T1", @@ -720,9 +741,10 @@ def test_the_bootstrap_is_not_run_when_the_uplift_is_nan(self) -> None: ) diag = out.uncertainty_diagnostics assert diag is not None - assert (diag["n_blocks"] == 0).all() - assert diag["frac_resamples_finite"].isna().all() - assert (diag["n_baseline_records"] == 0).all() + legs = diag[diag["component"] != "combined"] + assert (legs["n_blocks"] == 0).all() + assert legs["frac_resamples_finite"].isna().all() + assert (legs["n_baseline_records"] == 0).all() class TestUncertaintyReproducibility: @@ -754,7 +776,8 @@ def test_an_uncertainty_csv_is_written(self, tmp_path: Path) -> None: assert len(written) == 1 frame = pd.read_csv(written[0]) assert "n_upgraded_records" in frame.columns - assert len(frame) == 7 # overall + six power bins + # overall + six power bins, per leg and for the blend + assert frame.groupby("component").size().to_dict() == {"sum": 7, "block_mean": 7, "combined": 7} # --- labeled_rows: the row selection the estimate actually used --------------------------------- @@ -794,7 +817,9 @@ def test_one_row_per_test_turbine_record(self) -> None: def test_used_reproduces_the_methods_own_used_mask(self) -> None: scada, schedule = _noisy_toggle_case() - method = ToggleSpecialistMethod(columns=_COLUMNS, conditions=("power",), rated_power_kw=_RATED_KW) + method = ToggleSpecialistMethod( + columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + ) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) out = method.estimate(mi) assert out.labeled_rows is not None @@ -956,7 +981,7 @@ class TestExcludeRow: def test_flagged_test_rows_are_not_used(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -972,42 +997,52 @@ def test_excluding_corrupted_rows_restores_the_known_uplift(self) -> None: spoiled = corrupted[_EXCLUDE_COL].to_numpy(dtype=bool) & (corrupted[_TURBINE_COL] == "T1").to_numpy() corrupted.loc[spoiled, _POWER_COL] *= 0.5 - naive = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + naive = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert filtered.p50_overall == pytest.approx(0.05, abs=1e-9) assert abs(naive.p50_overall - 0.05) > abs(filtered.p50_overall - 0.05) - def test_flagged_reference_rows_drop_the_timestamp(self) -> None: - """A flagged reference row (curtailed, iced, parked) removes the timestamp like a reference outage.""" + def test_flagged_reference_rows_leave_the_reference_leg_like_an_outage(self, tmp_path: Path) -> None: + """A flagged reference row (curtailed, iced, parked) removes the timestamp from the reference leg. + + The block leg uses no references, so it keeps the timestamp and the headline ``used`` stays. + """ scada, schedule = _noisy_toggle_case() + # flag both references, so the flags reach whichever subset the reference leg keeps flagged = _flag(scada, turbine="R1", every=3) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + flagged[_EXCLUDE_COL] = ( + flagged[_EXCLUDE_COL].to_numpy() | _flag(scada, turbine="R2", every=3)[_EXCLUDE_COL].to_numpy() + ) + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None - is_ref = (flagged[_TURBINE_COL] == "R1").to_numpy() - flagged_ts = flagged.index[is_ref & flagged[_EXCLUDE_COL].to_numpy(dtype=bool)] + is_ref = (flagged[_TURBINE_COL] != "T1").to_numpy() + flagged_ts = flagged.index[is_ref & flagged[_EXCLUDE_COL].to_numpy(dtype=bool)].unique() used = out.labeled_rows["used"].astype(bool) - assert not used.loc[flagged_ts].any() - assert used.drop(flagged_ts).any() + assert used.loc[flagged_ts].all() + kept = _read_only_csv(tmp_path, "selection").set_index(["component", "stage", "segment"])["n_kept"] + for segment in ("baseline", "upgraded"): + assert kept[("sum", "exclude_row", segment)] < kept[("sum", "filters", segment)] + assert kept[("block_mean", "exclude_row", segment)] == kept[("block_mean", "filters", segment)] def test_nan_in_the_exclude_column_is_rejected(self) -> None: """NaN is not "excluded": the schema contract says the column is never NaN, so say so loudly.""" scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=7, value=np.nan) with pytest.raises(ValueError, match=_EXCLUDE_COL): - ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) def test_unset_role_excludes_nothing(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1017,7 +1052,7 @@ def test_unset_role_excludes_nothing(self) -> None: def test_absent_column_excludes_nothing(self) -> None: """The role names a column the frame does not carry: skip, do not raise.""" scada, schedule = _noisy_toggle_case() - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1032,9 +1067,9 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: """ scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, out_dir=tmp_path, save_plots=True).estimate( - MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + ToggleSpecialistMethod( + columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=True + ).estimate(MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) names = {p.name for p in (run_dir / "plots").rglob("*.png")} assert "excluded_row_fraction.png" in names @@ -1042,7 +1077,7 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: def test_no_exclusion_plots_when_nothing_is_excluded(self, tmp_path: Path) -> None: scada, schedule = _noisy_toggle_case() - ToggleSpecialistMethod(columns=_COLUMNS, out_dir=tmp_path, save_plots=True).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=True).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) @@ -1078,7 +1113,7 @@ def _fixture() -> tuple[pd.DataFrame, ToggleSchedule]: @staticmethod def _estimate(scada: pd.DataFrame, **kwargs: object) -> float: - method = ToggleSpecialistMethod(columns=_COLUMNS) + method = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE) return method.estimate(MethodInput(scada_df=scada, **kwargs)).p50_overall def test_only_offered_references_are_used(self) -> None: @@ -1202,6 +1237,7 @@ def _block_excluded_case() -> tuple[pd.DataFrame, ToggleSchedule, pd.DatetimeInd @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + kwargs.setdefault("reference_block", _CYCLE) return ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -1268,7 +1304,7 @@ def test_pairing_gap_plot_is_written(tmp_path: Path, gap: pd.Timedelta | None) - assert len(list(tmp_path.rglob("*_pairing_gap.png"))) == 1 -# --- reference modes: block_mean (zero references) and block_scaled ------------------------------ +# --- the two legs, their blend and the decisions on permuted labels ----------------------------- _BLOCK = pd.Timedelta(minutes=40) # one full 20-min on / 20-min off cycle = 4 rows of 10 min @@ -1285,7 +1321,7 @@ def _block_case( Test power is ``level_b * shape_i``, times ``1 + uplift`` when treated, where ``shape`` is a within-block ramp of slope ``ramp`` per row (flat by default: a ramp in phase with the toggle - is exactly the drift ``block_mean`` cannot cancel). With ``ref_scale`` (one factor per block) a + is exactly the drift the block leg cannot cancel). With ``ref_scale`` (one factor per block) a reference ``R1 = ref_scale_b * level_b * shape_i`` is added: it tracks the test turbine inside every block but its ratio to the test changes from block to block. """ @@ -1303,33 +1339,58 @@ def _block_case( return _scada(turbines, idx), schedule -class TestReferenceModes: - def test_block_mean_recovers_the_uplift_with_no_references(self) -> None: +def _two_reference_case(noise_frac: float = 0.3, seed: int = 5) -> tuple[pd.DataFrame, ToggleSchedule]: + """``_block_case`` with a perfect tracker ``R1`` plus a noisy tracker ``R2``: the union is worse than R1 alone.""" + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5), seed=seed) + rng = np.random.default_rng(seed) + r2 = scada[scada[_TURBINE_COL] == "R1"].copy() + r2[_TURBINE_COL] = "R2" + r2[_POWER_COL] = r2[_POWER_COL].to_numpy() * (1.0 + noise_frac * rng.standard_normal(len(r2))) + return pd.concat([scada, r2]), schedule + + +def _legs(out: MethodOutput) -> set[str]: + return set(out.uncertainty_diagnostics["component"]) + + +class TestReferenceBlock: + """``reference_block`` is the caller's statement of one toggle cycle: required, never inferred.""" + + def test_it_is_required(self) -> None: + with pytest.raises(TypeError, match="reference_block"): + ToggleSpecialistMethod(columns=_COLUMNS) # type: ignore[call-arg] + + def test_it_is_independent_of_the_pairing_gap(self) -> None: + scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) + out = _estimate(scada, schedule, reference_block=_BLOCK, pairing_max_gap=pd.Timedelta(minutes=10)) + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + + def test_a_block_off_the_timebase_grid_is_refused(self) -> None: + scada, schedule = _block_case(n_blocks=10, ref_scale=np.ones(10)) + with pytest.raises(ValueError, match="reference_block"): + _estimate(scada, schedule, reference_block=pd.Timedelta(minutes=25)) + + +class TestBlockLeg: + """The block leg: the test turbine against its own block's mean power, needing no references.""" + + def test_recovers_the_uplift_with_no_references(self) -> None: scada, schedule = _block_case(uplift=0.04) - out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + out = _estimate(scada, schedule, reference_block=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) assert np.isfinite(out.sigma_overall) + assert _legs(out) == {"block_mean"} - def test_block_mean_ignores_a_reference_outage(self) -> None: - """Zero-reference mode must not pay for references it does not use.""" + def test_ignores_a_reference_outage(self) -> None: + """A reference the leg does not use must not cost it rows; the estimate is then this leg alone.""" scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() scada.loc[is_ref, _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + out = _estimate(scada, schedule, reference_block=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) assert out.labeled_rows is not None assert out.labeled_rows["used"].astype(bool).all() - - def test_block_scaled_recovers_the_uplift_when_the_reference_ratio_drifts_between_blocks(self) -> None: - rng = np.random.default_rng(1) - scada, schedule = _block_case(uplift=0.04, ramp=0.1, ref_scale=rng.uniform(0.3, 3.0, 120)) - out = _estimate(scada, schedule, reference_mode="block_scaled", reference_block=_BLOCK) - assert out.p50_overall == pytest.approx(0.04, abs=1e-9) - - def test_block_length_defaults_to_the_pairing_gap(self) -> None: - scada, schedule = _block_case(uplift=0.04) - out = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=_BLOCK) - assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + assert _legs(out) == {"block_mean"} def test_a_block_with_one_state_is_dropped_and_accounted(self) -> None: scada, schedule = _block_case(uplift=0.04) @@ -1337,7 +1398,7 @@ def test_a_block_with_one_state_is_dropped_and_accounted(self) -> None: block_rows = idx[40:44] # the 11th block: two on/off cycles off_rows = block_rows[~np.asarray(treated_mask(block_rows, schedule))] scada.loc[scada.index.isin(off_rows), _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) + out = _estimate(scada, schedule, reference_block=_BLOCK) assert out.labeled_rows is not None # NaN (a timestamp absent from the pivot) must read as unused, so compare rather than cast used = out.labeled_rows["used"] == True # noqa: E712 @@ -1347,7 +1408,7 @@ def test_a_block_with_one_state_is_dropped_and_accounted(self) -> None: assert kept["block", "upgraded"] == kept["pairing", "upgraded"] - 2 assert kept["block", "baseline"] == kept["pairing", "baseline"] - def test_sum_mode_reports_a_block_stage_that_keeps_everything(self) -> None: + def test_the_reference_leg_keeps_every_block_without_flags(self) -> None: scada, schedule = _noisy_toggle_case(n=200) out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] @@ -1355,58 +1416,61 @@ def test_sum_mode_reports_a_block_stage_that_keeps_everything(self) -> None: assert list(out.selection_accounting["stage"].unique()) == stages assert kept["block", "baseline"] == kept["pairing", "baseline"] - def test_unknown_mode_raises(self) -> None: - with pytest.raises(ValueError, match="reference_mode"): - ToggleSpecialistMethod(columns=_COLUMNS, reference_mode="bogus") - def test_a_block_mode_without_a_block_length_raises(self) -> None: - scada, schedule = _block_case(n_blocks=10, ref_scale=np.ones(10)) - with pytest.raises(ValueError, match="reference_block"): - _estimate(scada, schedule, reference_mode="block_scaled") +class TestBlend: + """The estimate blends the reference and block legs by minimum-variance weights.""" - -class TestCombinedMode: - """``reference_mode="combined"``: inverse-variance blend of the ``sum`` and ``block_mean`` estimates.""" - - def test_recovers_the_uplift_when_both_components_are_exact(self) -> None: + def test_recovers_the_uplift_when_both_legs_are_exact(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) - out = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) + out = _estimate(scada, schedule, reference_block=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + assert _legs(out) == {"sum", "block_mean", "combined"} - def test_falls_back_to_block_mean_when_the_references_are_out(self) -> None: - scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5), seed=3) - scada.loc[(scada[_TURBINE_COL] == "R1").to_numpy(), _POWER_COL] = np.nan - combined = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) - alone = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) - assert combined.p50_overall == pytest.approx(alone.p50_overall) - assert combined.sigma_overall == pytest.approx(alone.sigma_overall) - - def test_sigma_lies_between_the_naive_blend_and_the_better_component(self) -> None: - scada, schedule = _noisy_toggle_case() - gap = pd.Timedelta(minutes=20) - combined = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=gap) - total = _estimate(scada, schedule, reference_mode="sum", pairing_max_gap=gap) - block = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=gap) - naive = 1.0 / np.sqrt(1.0 / total.sigma_overall**2 + 1.0 / block.sigma_overall**2) - assert naive <= combined.sigma_overall <= max(total.sigma_overall, block.sigma_overall) - - def test_diagnostics_name_the_components(self) -> None: - scada, schedule = _noisy_toggle_case(n=400) - out = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=pd.Timedelta(minutes=20)) - assert set(out.uncertainty_diagnostics["component"]) == {"sum", "block_mean", "combined"} - combined = out.uncertainty_diagnostics[out.uncertainty_diagnostics["component"] == "combined"] - assert np.isfinite(combined.loc[combined["condition_bin"] == "overall", "weight_sum"]).all() - - def test_used_rows_are_the_union_of_the_components(self) -> None: + def test_used_rows_are_the_union_of_the_legs(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() scada.loc[is_ref & (np.arange(len(scada)) % 7 == 0), _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_mode="combined", reference_block=_BLOCK) - total = _estimate(scada, schedule, reference_mode="sum", reference_block=_BLOCK) - used_combined = out.labeled_rows["used"] == True # noqa: E712 - NaN (absent timestamp) must read as unused - used_sum = total.labeled_rows["used"] == True # noqa: E712 - assert used_combined.sum() > used_sum.sum() - assert used_combined.all() + out = _estimate(scada, schedule, reference_block=_BLOCK) + used = out.labeled_rows["used"] == True # noqa: E712 - NaN (absent timestamp) must read as unused + assert used.all() + # The accounting is the reference leg's, which the reference outage did cost rows. + kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + assert kept["block", "baseline"] + kept["block", "upgraded"] < used.sum() + + def test_diagnostics_carry_both_legs_and_the_blend(self) -> None: + scada, schedule = _noisy_toggle_case(n=400) + out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) + assert _legs(out) == {"sum", "block_mean", "combined"} + diag = out.uncertainty_diagnostics + head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] + assert 0.0 <= head["weight_sum"] <= 1.0 + assert np.isfinite(head["decision_sigma_sum"]) + assert np.isfinite(head["decision_sigma_block_mean"]) + + def test_the_blend_is_the_weighted_legs_and_its_sigma_comes_from_the_actual_bootstraps( + self, tmp_path: Path + ) -> None: + scada, schedule = _noisy_toggle_case(n=600) + out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20), out_dir=tmp_path) + results = _read_only_csv(tmp_path, "results").iloc[0] + diag = out.uncertainty_diagnostics + head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] + legs = { + m: diag[(diag["component"] == m) & (diag["condition_bin"] == "overall")].iloc[0] + for m in ("sum", "block_mean") + } + w = head["weight_sum"] + assert out.p50_overall == pytest.approx( + w * results["uplift_frc_sum"] + (1 - w) * results["uplift_frc_block_mean"] + ) + sa, sb, r = legs["sum"]["sigma"], legs["block_mean"]["sigma"], head["correlation"] + assert sa == pytest.approx(results["uplift_sigma_frc_sum"]) + assert sb == pytest.approx(results["uplift_sigma_frc_block_mean"]) + variance = w**2 * sa**2 + (1 - w) ** 2 * sb**2 + 2 * w * (1 - w) * r * sa * sb + assert out.sigma_overall == pytest.approx(np.sqrt(variance)) + # The weights were judged on the permuted sigmas, which are not the reported ones. + assert results["decision_sigma_frc_sum"] != results["uplift_sigma_frc_sum"] + assert results["decision_sigma_frc_block_mean"] != results["uplift_sigma_frc_block_mean"] _BLOCK_COL = "cycle_bad" @@ -1422,7 +1486,7 @@ def _flag_block(scada: pd.DataFrame, *, turbine: str, at: pd.Timestamp) -> pd.Da class TestExcludeBlock: - """``columns.exclude_block``: a flagged row removes its whole reference block, in every mode.""" + """``columns.exclude_block``: a flagged row removes its whole reference block, in both legs.""" @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: @@ -1430,7 +1494,7 @@ def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> Met MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - def test_a_flagged_test_row_removes_its_whole_block_in_sum_mode(self) -> None: + def test_a_flagged_test_row_removes_its_whole_block(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) idx = pd.DatetimeIndex(pd.unique(scada.index)) block_5 = idx[20:24] @@ -1444,21 +1508,18 @@ def test_a_flagged_test_row_removes_its_whole_block_in_sum_mode(self) -> None: for segment in ("baseline", "upgraded"): assert acc.loc[("block", segment)] == acc.loc[("pairing", segment)] - 2 - def test_a_flagged_reference_row_removes_the_block_too(self) -> None: + def test_a_flagged_reference_row_removes_the_block_from_the_reference_leg(self) -> None: + """The block leg uses no references, so it keeps the block and the headline ``used`` stays.""" scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) idx = pd.DatetimeIndex(pd.unique(scada.index)) block_7 = idx[28:32] out = self._run(_flag_block(scada, turbine="R1", at=block_7[3]), schedule, reference_block=_BLOCK) assert out.labeled_rows is not None - used = out.labeled_rows["used"].astype(bool) - assert not used.loc[block_7].any() - assert used.drop(block_7).all() - - def test_a_flag_with_no_block_length_is_refused(self) -> None: - scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) - idx = pd.DatetimeIndex(pd.unique(scada.index)) - with pytest.raises(ValueError, match="exclude_block"): - self._run(_flag_block(scada, turbine="T1", at=idx[21]), schedule) + assert out.labeled_rows["used"].astype(bool).all() + acc = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] + for segment in ("baseline", "upgraded"): + assert acc.loc[("block", segment)] == acc.loc[("pairing", segment)] - 2 + assert out.p50_overall == pytest.approx(0.04, abs=1e-9) def test_no_flags_change_nothing(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) @@ -1469,92 +1530,55 @@ def test_no_flags_change_nothing(self) -> None: assert with_role.sigma_overall == pytest.approx(without.sigma_overall) -def _two_reference_case(noise_frac: float = 0.3, seed: int = 5) -> tuple[pd.DataFrame, ToggleSchedule]: - """``_block_case`` with a perfect tracker ``R1`` plus a noisy tracker ``R2``: the union is worse than R1 alone.""" - scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5), seed=seed) - rng = np.random.default_rng(seed) - r2 = scada[scada[_TURBINE_COL] == "R1"].copy() - r2[_TURBINE_COL] = "R2" - r2[_POWER_COL] = r2[_POWER_COL].to_numpy() * (1.0 + noise_frac * rng.standard_normal(len(r2))) - return pd.concat([scada, r2]), schedule - +class TestReferenceSubsets: + """The reference leg takes the reference subset with the smallest label-permuted sigma.""" -class TestReferenceChoice: - """``reference_choice="min_sigma"``: the sum leg takes the reference subset with the smallest headline sigma.""" - - def test_the_default_keeps_every_reference(self, tmp_path: Path) -> None: + def test_picks_the_subset_with_the_smallest_permuted_sigma(self, tmp_path: Path) -> None: scada, schedule = _two_reference_case() - _estimate(scada, schedule, out_dir=tmp_path) - results = _read_only_csv(tmp_path, "results") - assert results.loc[0, "reference_choice"] == "all" - assert results.loc[0, "refs_used"] == "R1;R2" - assert results.loc[0, "n_refs"] == 2 - assert results.loc[0, "n_refs_available"] == 2 - - def test_min_sigma_picks_the_subset_with_the_smallest_sigma(self, tmp_path: Path) -> None: - scada, schedule = _two_reference_case() - out = _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path) + out = _estimate(scada, schedule, out_dir=tmp_path) results = _read_only_csv(tmp_path, "results") assert results.loc[0, "refs_used"] == "R1" assert results.loc[0, "n_refs"] == 1 assert results.loc[0, "n_refs_available"] == 2 assert out.p50_overall == pytest.approx(0.04, abs=1e-9) - union = _estimate(scada, schedule) - assert out.sigma_overall < union.sigma_overall + subsets = _read_only_csv(tmp_path, "reference_subsets") + assert subsets["decision_sigma_frc"].notna().all() + chosen = subsets.loc[subsets["chosen"], "refs"].item() + assert chosen == subsets.loc[subsets["decision_sigma_frc"].idxmin(), "refs"] + assert chosen == "R1" # the perfect tracker is the least noisy on any basis - def test_every_subset_is_written_with_the_winner_marked(self, tmp_path: Path) -> None: + def test_every_subset_is_written_and_only_the_winner_has_an_actual_sigma(self, tmp_path: Path) -> None: scada, schedule = _two_reference_case() - _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path) - subsets = _read_only_csv(tmp_path, "reference_subsets") - assert sorted(subsets["refs"]) == ["R1", "R1;R2", "R2"] + _estimate(scada, schedule, out_dir=tmp_path) + subsets = _read_only_csv(tmp_path, "reference_subsets").set_index("refs") + assert sorted(subsets.index) == ["R1", "R1;R2", "R2"] assert subsets["chosen"].sum() == 1 - assert subsets.loc[subsets["chosen"], "refs"].item() == "R1" - assert subsets["uplift_sigma_frc"].notna().all() assert (subsets["n_used_timestamps"] > 0).all() + assert subsets["uplift_sigma_frc"].notna().sum() == 1 + assert np.isfinite(subsets.loc["R1", "uplift_sigma_frc"]) - def test_combined_blends_the_chosen_subset(self, tmp_path: Path) -> None: + def test_the_reported_sigma_is_the_actual_one_of_the_chosen_subset(self) -> None: scada, schedule = _two_reference_case() - out = _estimate( - scada, - schedule, - reference_mode="combined", - reference_choice="min_sigma", - reference_block=_BLOCK, - out_dir=tmp_path, - ) - results = _read_only_csv(tmp_path, "results") - assert results.loc[0, "refs_used"] == "R1" - assert set(out.uncertainty_diagnostics["component"]) == {"sum", "block_mean", "combined"} - assert out.p50_overall == pytest.approx(0.04, abs=1e-9) + chosen = _estimate(scada, schedule) + only_r1 = _estimate(scada[scada[_TURBINE_COL] != "R2"], schedule) + assert chosen.p50_overall == pytest.approx(only_r1.p50_overall) + assert chosen.sigma_overall == pytest.approx(only_r1.sigma_overall) - def test_block_mean_ignores_the_choice(self) -> None: + def test_the_permuted_sigma_measures_the_same_noise_as_the_actual_one(self, tmp_path: Path) -> None: + """For the noisy reference the two sigmas agree; they differ only in whether the on/off split is real.""" scada, schedule = _two_reference_case() - chosen = _estimate( - scada, schedule, reference_mode="block_mean", reference_choice="min_sigma", reference_block=_BLOCK - ) - plain = _estimate(scada, schedule, reference_mode="block_mean", reference_block=_BLOCK) - assert chosen.p50_overall == pytest.approx(plain.p50_overall) - assert chosen.sigma_overall == pytest.approx(plain.sigma_overall) - - def test_an_unknown_choice_is_refused(self) -> None: - with pytest.raises(ValueError, match="reference_choice"): - ToggleSpecialistMethod(columns=_COLUMNS, reference_choice="best") - + _estimate(scada, schedule, out_dir=tmp_path / "both") + _estimate(scada[scada[_TURBINE_COL] != "R1"], schedule, out_dir=tmp_path / "r2") + permuted = _read_only_csv(tmp_path / "both", "reference_subsets").set_index("refs")["decision_sigma_frc"] + actual_r2 = _read_only_csv(tmp_path / "r2", "results").loc[0, "uplift_sigma_frc_sum"] + assert permuted["R2"] == pytest.approx(actual_r2, rel=0.3) + assert permuted["R1"] < permuted["R2"] -class TestDecisionBasis: - """``decision_basis="permuted"``: the subset choice and the blend weights come from label-permuted bootstraps.""" - def test_an_unknown_basis_is_refused(self) -> None: - with pytest.raises(ValueError, match="decision_basis"): - ToggleSpecialistMethod(columns=_COLUMNS, decision_basis="lucky") +class TestPermutedLabels: + """``permute_labels_within_blocks``: the same campaign with the on/off labels shuffled per block.""" - def test_actual_is_the_default(self, tmp_path: Path) -> None: - scada, schedule = _two_reference_case() - _estimate(scada, schedule, out_dir=tmp_path) - results = _read_only_csv(tmp_path, "results") - assert results.loc[0, "decision_basis"] == "actual" - - def test_permuted_labels_keep_every_blocks_on_off_counts(self) -> None: + def test_every_block_keeps_its_on_off_counts(self) -> None: rng = np.random.default_rng(0) n = 40 upgraded = np.arange(n) % 4 < 2 # two on, two off per block of four @@ -1573,7 +1597,7 @@ def test_permuted_labels_keep_every_blocks_on_off_counts(self) -> None: assert (up_p & rows).sum() == (upgraded & rows).sum() assert (base_p & rows).sum() == (baseline & rows).sum() - def test_permuted_labels_without_blocks_are_shuffled_globally(self) -> None: + def test_without_blocks_the_labels_are_shuffled_globally(self) -> None: rng = np.random.default_rng(1) upgraded = np.arange(12) % 2 == 0 up_p, base_p = toggle_specialist.permute_labels_within_blocks( @@ -1581,58 +1605,3 @@ def test_permuted_labels_without_blocks_are_shuffled_globally(self) -> None: ) assert up_p.sum() == upgraded.sum() assert np.array_equal(up_p, ~base_p) - - def test_min_sigma_ranks_the_subsets_by_the_permuted_sigma(self, tmp_path: Path) -> None: - scada, schedule = _two_reference_case() - out = _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted", out_dir=tmp_path) - subsets = _read_only_csv(tmp_path, "reference_subsets") - assert subsets["decision_sigma_frc"].notna().all() - chosen = subsets.loc[subsets["chosen"], "refs"].item() - assert chosen == subsets.loc[subsets["decision_sigma_frc"].idxmin(), "refs"] - assert chosen == "R1" # the perfect tracker is the least noisy on any basis - assert out.p50_overall == pytest.approx(0.04, abs=1e-9) - results = _read_only_csv(tmp_path, "results") - assert results.loc[0, "decision_basis"] == "permuted" - assert results.loc[0, "refs_used"] == "R1" - - def test_the_reported_sigma_is_the_actual_one_of_the_chosen_subset(self) -> None: - scada, schedule = _two_reference_case() - chosen = _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted") - only_r1 = _estimate(scada[scada[_TURBINE_COL] != "R2"], schedule) - assert chosen.p50_overall == pytest.approx(only_r1.p50_overall) - assert chosen.sigma_overall == pytest.approx(only_r1.sigma_overall) - # The permuted sigma is a decision aid, never the answer: it is not the reported sigma. - assert chosen.sigma_overall > 0 - - def test_the_permuted_sigma_measures_the_same_noise_as_the_actual_one(self, tmp_path: Path) -> None: - """For a noisy reference the two sigmas agree; they differ only in whether the on/off split is the real one.""" - scada, schedule = _two_reference_case() - _estimate(scada, schedule, reference_choice="min_sigma", out_dir=tmp_path / "actual") - _estimate(scada, schedule, reference_choice="min_sigma", decision_basis="permuted", out_dir=tmp_path / "perm") - actual = _read_only_csv(tmp_path / "actual", "reference_subsets").set_index("refs")["uplift_sigma_frc"] - permuted = _read_only_csv(tmp_path / "perm", "reference_subsets").set_index("refs")["decision_sigma_frc"] - for refs in ("R2", "R1;R2"): # R1 alone is a perfect tracker: its actual sigma is exactly zero - assert permuted[refs] == pytest.approx(actual[refs], rel=0.3) - assert permuted["R1"] < permuted["R2"] - - def test_combined_weights_come_from_the_permuted_sigmas_but_the_sigma_is_actual(self) -> None: - scada, schedule = _noisy_toggle_case(n=600) - gap = pd.Timedelta(minutes=20) - out = _estimate(scada, schedule, reference_mode="combined", pairing_max_gap=gap, decision_basis="permuted") - diag = out.uncertainty_diagnostics - head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] - legs = { - m: diag[(diag["component"] == m) & (diag["condition_bin"] == "overall")].iloc[0] - for m in ("sum", "block_mean") - } - w = head["weight_sum"] - assert 0.0 <= w <= 1.0 - total = _estimate(scada, schedule, reference_mode="sum", pairing_max_gap=gap) - block = _estimate(scada, schedule, reference_mode="block_mean", pairing_max_gap=gap) - assert out.p50_overall == pytest.approx(w * total.p50_overall + (1 - w) * block.p50_overall) - sa, sb, r = legs["sum"]["sigma"], legs["block_mean"]["sigma"], head["correlation"] - assert sa == pytest.approx(total.sigma_overall) - variance = w**2 * sa**2 + (1 - w) ** 2 * sb**2 + 2 * w * (1 - w) * r * sa * sb - assert out.sigma_overall == pytest.approx(np.sqrt(variance)) - assert np.isfinite(head["decision_sigma_sum"]) - assert np.isfinite(head["decision_sigma_block_mean"]) From e652bd5828a71e851a125160eac1342462b826c7 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 12:48:05 +0100 Subject: [PATCH 13/18] Fix the toggle placebo end-to-end tests - The placebo fixture is 10-minute SCADA: its 100-minute toggle cycle cannot tile an hourly grid, which the required reference_block now refuses. - A component with no resample spread takes the default correlation rather than tripping np.corrcoef's divide warning (identical-turbine placebo). - ToggleSpecialist hands the harness one uncertainty_diagnostics row per (condition, condition_bin), each leg's columns suffixed _; the per-leg rows clashed with the harness's sigma column and its one-row-per-key rule. The uncertainty CSV keeps the per-leg rows. Co-Authored-By: Claude Opus 5.5 --- benchmarking/baselines/block_bootstrap.py | 5 ++- benchmarking/baselines/toggle_specialist.py | 11 ++++- .../baselines/test_block_bootstrap.py | 5 +++ .../baselines/test_toggle_specialist.py | 41 +++++++++++++++---- .../campaigns/test_placebo_end_to_end.py | 8 ++-- 5 files changed, 56 insertions(+), 14 deletions(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 294433ed..02f08a11 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -170,7 +170,10 @@ def _paired_correlation(res_a: npt.NDArray[np.float64], res_b: npt.NDArray[np.fl paired = np.isfinite(res_a[:n_pairs]) & np.isfinite(res_b[:n_pairs]) if paired.sum() < _MIN_RESAMPLES_FOR_SPREAD: return _DEFAULT_COMPONENT_CORRELATION - r = float(np.corrcoef(res_a[:n_pairs][paired], res_b[:n_pairs][paired])[0, 1]) + pa, pb = res_a[:n_pairs][paired], res_b[:n_pairs][paired] + if np.ptp(pa) == 0.0 or np.ptp(pb) == 0.0: + return _DEFAULT_COMPONENT_CORRELATION + r = float(np.corrcoef(pa, pb)[0, 1]) return r if math.isfinite(r) else _DEFAULT_COMPONENT_CORRELATION diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 9b151b11..feaf33e9 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -415,7 +415,7 @@ def estimate(self, mi: MethodInput) -> MethodOutput: p50_overall=float(est.uplift), p50_by_condition=est.per_bin, sigma_overall=est.sigma_overall, - uncertainty_diagnostics=est.diagnostics, + uncertainty_diagnostics=_harness_diagnostics(est.diagnostics), labeled_rows=self._labeled_rows(mi, wide=wide, test=test, used=est.used, rows=rows, label=est.label), selection_accounting=est.accounting, ) @@ -1202,6 +1202,15 @@ def _cell_sigma(boot: BootstrapResult | None, cell: str) -> float: return boot.cells[cell].sigma +def _harness_diagnostics(diagnostics: pd.DataFrame) -> pd.DataFrame: + """One row per ``(condition, condition_bin)``, each leg's columns suffixed ``_``.""" + keys = ["condition", "condition_bin"] + values = [c for c in diagnostics.columns if c not in (*keys, "component")] + wide = diagnostics.pivot(index=keys, columns="component", values=values).dropna(axis=1, how="all") # noqa: PD010 - unique keys, nothing to aggregate + wide.columns = [f"{value}_{component}" for value, component in wide.columns] + return wide.reset_index() + + def _uncertainty_diagnostics( boot: BootstrapResult | None, *, diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index 91c9dfa9..418a72e1 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -463,6 +463,11 @@ def test_missing_resamples_assume_a_half_correlation(self) -> None: assert out.correlation == 0.5 assert out.sigma == pytest.approx(np.sqrt((1 + 0.5) / 2)) + def test_resamples_without_spread_assume_a_half_correlation(self) -> None: + # identical turbines: every resample of a leg reads exactly the same uplift + out = combine_estimates((0.0, 1.0, np.zeros(100)), (0.0, 1.0, np.linspace(-1, 1, 100))) + assert out.correlation == 0.5 + class TestASparseCellWithNoFallbackReportsNaN: """Two records in a long campaign: every resample that sees them is identical, so the bootstrap's diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index ed6929ce..bc40bf3d 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -485,7 +485,7 @@ def test_no_reference_is_the_block_leg_alone(self, caplog: pytest.LogCaptureFixt out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - assert set(out.uncertainty_diagnostics["component"]) == {"block_mean"} + assert _legs(out) == {"block_mean"} assert "no reference turbines" in caplog.text def test_no_used_baseline_returns_nan(self) -> None: @@ -629,6 +629,22 @@ def _noisy_toggle_case(n: int = 2016, *, uplift: float = 0.03, noise_frac: float return scada, schedule +_LEG_NAMES = ("sum", "block_mean", "combined") + + +def _long_diagnostics(out: MethodOutput) -> pd.DataFrame: + """The harness frame back in per-leg rows: one row per (component, condition, condition_bin).""" + wide = out.uncertainty_diagnostics + assert wide is not None + frames = [] + for leg in _LEG_NAMES: + renamed = {c: c.removesuffix(f"_{leg}") for c in wide.columns if c.endswith(f"_{leg}")} + if renamed: + leg_rows = wide[["condition", "condition_bin", *renamed]].rename(columns=renamed) + frames.append(leg_rows.dropna(subset=list(renamed.values()), how="all").assign(component=leg)) + return pd.concat(frames, ignore_index=True) + + def _estimate(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: kwargs.setdefault("reference_block", _CYCLE) return ToggleSpecialistMethod(columns=_COLUMNS, **kwargs).estimate( @@ -653,10 +669,18 @@ def test_every_reported_bin_carries_a_sigma(self) -> None: assert len(populated) > 0 assert populated["sigma_uplift"].notna().all() + def test_the_harness_frame_has_one_row_per_cell_and_no_reserved_column(self) -> None: + scada, schedule = _noisy_toggle_case() + diag = _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW).uncertainty_diagnostics + assert diag is not None + assert not diag.duplicated(["condition", "condition_bin"]).any() + assert not set(diag.columns) & {"sigma", "estimate", "truth"} + assert {"sigma_sum", "sigma_block_mean", "sigma_combined"} <= set(diag.columns) + def test_diagnostics_cover_the_headline_and_every_bin(self) -> None: scada, schedule = _noisy_toggle_case() out = _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW) - diag = out.uncertainty_diagnostics + diag = _long_diagnostics(out) assert diag is not None assert (diag["condition"] == "overall").sum() == diag["component"].nunique() assert set(diag.columns) >= { @@ -686,7 +710,7 @@ def _candidates(tmp_path: Path, **kwargs: object) -> tuple[pd.Series, float, pd. out = _estimate(scada, schedule, out_dir=tmp_path, **kwargs) subsets = _read_only_csv(tmp_path, "reference_subsets").set_index("refs")["uplift_frc"] block = _read_only_csv(tmp_path, "results").loc[0, "uplift_frc_block_mean"] - diag = out.uncertainty_diagnostics.set_index(["component", "condition_bin"]) + diag = _long_diagnostics(out).set_index(["component", "condition_bin"]) return subsets, block, diag def test_block_length_moves_the_bootstrap_and_no_uplift(self, tmp_path: Path) -> None: @@ -739,11 +763,12 @@ def test_the_bootstrap_is_not_run_when_the_uplift_is_nan(self) -> None: turbine_col=_TURBINE_COL, ) ) - diag = out.uncertainty_diagnostics + diag = _long_diagnostics(out) assert diag is not None legs = diag[diag["component"] != "combined"] assert (legs["n_blocks"] == 0).all() - assert legs["frac_resamples_finite"].isna().all() + # no resample ran, so no leg reports a finite fraction (the harness frame drops the empty column) + assert "frac_resamples_finite" not in legs or legs["frac_resamples_finite"].isna().all() assert (legs["n_baseline_records"] == 0).all() @@ -1350,7 +1375,7 @@ def _two_reference_case(noise_frac: float = 0.3, seed: int = 5) -> tuple[pd.Data def _legs(out: MethodOutput) -> set[str]: - return set(out.uncertainty_diagnostics["component"]) + return set(_long_diagnostics(out)["component"]) class TestReferenceBlock: @@ -1441,7 +1466,7 @@ def test_diagnostics_carry_both_legs_and_the_blend(self) -> None: scada, schedule = _noisy_toggle_case(n=400) out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) assert _legs(out) == {"sum", "block_mean", "combined"} - diag = out.uncertainty_diagnostics + diag = _long_diagnostics(out) head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] assert 0.0 <= head["weight_sum"] <= 1.0 assert np.isfinite(head["decision_sigma_sum"]) @@ -1453,7 +1478,7 @@ def test_the_blend_is_the_weighted_legs_and_its_sigma_comes_from_the_actual_boot scada, schedule = _noisy_toggle_case(n=600) out = _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20), out_dir=tmp_path) results = _read_only_csv(tmp_path, "results").iloc[0] - diag = out.uncertainty_diagnostics + diag = _long_diagnostics(out) head = diag[(diag["component"] == "combined") & (diag["condition_bin"] == "overall")].iloc[0] legs = { m: diag[(diag["component"] == m) & (diag["condition_bin"] == "overall")].iloc[0] diff --git a/tests/benchmarking/campaigns/test_placebo_end_to_end.py b/tests/benchmarking/campaigns/test_placebo_end_to_end.py index 2aebcb19..d10912a6 100644 --- a/tests/benchmarking/campaigns/test_placebo_end_to_end.py +++ b/tests/benchmarking/campaigns/test_placebo_end_to_end.py @@ -43,15 +43,15 @@ def placebo_scada(mode: str, *, wind_noise_sd: float, seed: int = 0) -> pd.DataFrame: - """Hourly SCADA over ``mode``'s placebo period: a shared wind signal plus per-turbine noise. + """10-minute SCADA over ``mode``'s placebo period: a shared wind signal plus per-turbine noise. :param mode: the placebo mode whose analysis period the frame spans :param wind_noise_sd: per-turbine wind-speed noise in m/s; 0 makes every turbine identical :param seed: RNG seed for the noise """ rng = np.random.default_rng(seed) - index = pd.date_range(*placebo_analysis_period(mode), freq="1h", tz="UTC", inclusive="left") - hours = np.arange(len(index), dtype=float) + index = pd.date_range(*placebo_analysis_period(mode), freq="10min", tz="UTC", inclusive="left") + hours = np.arange(len(index), dtype=float) / 6 # a slow seasonal swing plus a daily cycle, so the pre and post periods differ in wind resource shared_ws = 8.0 + 2.5 * np.sin(hours / (24 * 30)) + 1.5 * np.sin(hours / 12) frames = [] @@ -68,7 +68,7 @@ def placebo_scada(mode: str, *, wind_noise_sd: float, seed: int = 0) -> pd.DataF HOT_COLUMNS.wind_speed: ws, HOT_COLUMNS.wind_speed_sd: 0.8, HOT_COLUMNS.gen_rpm: 1400.0, - HOT_COLUMNS.availability: 3600.0, + HOT_COLUMNS.availability: 600.0, }, index=index, ) From 74216e2293fb151446b5d2a791630cb689a72976 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 13:46:53 +0100 Subject: [PATCH 14/18] ToggleSpecialistMethod power_band: keep only rows whose reference-derived power lies in a band power_band=(lo, hi] restricts each leg's used rows to those whose bin power (rho_label * ref_total, the state-neutral baseline power the bins use) lies in the band, and estimates the uplift over them. The cut is a "power_band" stage in the selection accounting and is written to the run config. A caller measuring the uplift only where a control table is followed uses it instead of cutting on the test turbine's own power, which the state moves. Co-Authored-By: Claude Opus 5.5 --- benchmarking/baselines/toggle_specialist.py | 22 ++++++++++-- .../baselines/test_toggle_specialist.py | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index feaf33e9..d6c9f313 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -313,6 +313,9 @@ class ToggleSpecialistMethod: ``columns.exclude_block``: one toggle cycle (the campaign's own on + off duration), a positive whole multiple of the timebase. It is the caller's to state, not inferred, and it is independent of ``pairing_max_gap``. + :param power_band: when set, ``(lo, hi]`` in kW: each leg keeps only the used rows whose + reference-derived baseline power (``rho_label * ref_total``, the power the bins use, so + state-neutral) lies in it, and estimates its uplift over those. ``None`` keeps every used row. """ columns: ColumnSchema @@ -328,6 +331,7 @@ class ToggleSpecialistMethod: bootstrap_seed: int = 0 pairing_max_gap: pd.Timedelta | None = None segment_imbalance_warning: float = 0.1 + power_band: tuple[float, float] | None = None def __post_init__(self) -> None: """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" @@ -461,14 +465,22 @@ def _component( mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids, both_states=mode == _BLOCK_LEG ) used = selection.blocked.to_numpy() - accounting = self._selection_accounting(selection, rows=rows) ref_total = self._reference_total(mode=mode, wide=wide, refs=refs, test_pw=test_pw, used=used, ids=ids) rho_base = _rho(test_pw, ref_total, used & baseline) rho_up = _rho(test_pw, ref_total, used & rows.upgraded) + rho_label = _rho_label(rho_base, rho_up) + in_band = None + if self.power_band is not None: + lo, hi = self.power_band + label = rho_label * ref_total + in_band = used & (label > lo) & (label <= hi) + used = in_band + rho_base = _rho(test_pw, ref_total, used & baseline) + rho_up = _rho(test_pw, ref_total, used & rows.upgraded) + accounting = self._selection_accounting(selection, rows=rows, power_band=in_band) recoverable = np.isfinite(rho_base) and rho_base != 0 and np.isfinite(rho_up) uplift = rho_up / rho_base - 1.0 if recoverable else np.nan - rho_label = _rho_label(rho_base, rho_up) per_bin = ( self._conditional_frame( @@ -838,7 +850,9 @@ def _selection( blocked=pd.Series(kept, index=wide.index), ) - def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) -> pd.DataFrame: + def _selection_accounting( + self, selection: _Selection, *, rows: ToggleRowSets, power_band: npt.NDArray[np.bool_] | None = None + ) -> pd.DataFrame: """Rows kept per segment after each stage, warning when a stage treats the segments unevenly. ``kept_fraction`` is relative to the segment's rows; ``stage_kept_fraction`` to the previous @@ -850,6 +864,7 @@ def _selection_accounting(self, selection: _Selection, *, rows: ToggleRowSets) - ("exclude_row", selection.not_excluded), ("pairing", selection.paired), ("block", selection.blocked), + *(() if power_band is None else (("power_band", pd.Series(power_band, index=selection.blocked.index)),)), ) records = [] for segment, in_segment in ((_BASELINE, rows.campaign_baseline), (_UPGRADED, rows.upgraded)): @@ -1069,6 +1084,7 @@ def _write_shared_diagnostics( "availability_col": self.columns.availability, "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), "reference_block": str(self.reference_block), + "power_band": None if self.power_band is None else list(self.power_band), "exclude_block_col": self.columns.exclude_block, } write_run_config(ctx, method_name=self.name, method_params=params) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index bc40bf3d..474e8263 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1378,6 +1378,42 @@ def _legs(out: MethodOutput) -> set[str]: return set(_long_diagnostics(out)["component"]) +class TestPowerBand: + """``power_band`` keeps the used rows whose reference-derived baseline power lies in ``(lo, hi]``.""" + + _BAND = (360.0, 840.0) # two whole 240 kW bins at 1200 kW rated + + @staticmethod + def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + return _estimate(scada, schedule, conditions=("power",), rated_power_kw=_RATED_KW, **kwargs) + + def test_every_used_row_lies_in_the_band(self) -> None: + out = self._run(*_varying_rho_case(n=2016, uplift=0.05), power_band=self._BAND) + assert out.labeled_rows is not None + used_bins = set(out.labeled_rows.loc[out.labeled_rows["used"].astype(bool), "power_bin"].astype(str)) + assert used_bins == {"(360.0, 600.0]", "(600.0, 840.0]"} + + def test_a_band_holding_every_row_changes_nothing(self) -> None: + scada, schedule = _noisy_toggle_case() + assert self._run(scada, schedule, power_band=(-1e9, 1e9)).p50_overall == pytest.approx( + self._run(scada, schedule).p50_overall, abs=1e-12 + ) + + def test_a_constant_uplift_reads_the_same_inside_the_band(self) -> None: + """The cut reads the reference-derived power, which the state cannot move, so it selects no state.""" + out = self._run(*_varying_rho_case(n=2016, uplift=0.05), power_band=self._BAND) + assert out.p50_overall == pytest.approx(0.05, abs=2e-3) + + def test_the_band_is_a_selection_stage_and_in_the_run_config(self, tmp_path: Path) -> None: + # the run config is written with the plots + self._run(*_varying_rho_case(n=2016, uplift=0.05), power_band=self._BAND, out_dir=tmp_path, save_plots=True) + stages = set(_read_only_csv(tmp_path, "selection")["stage"]) + assert "power_band" in stages + run_dir = next(p for p in tmp_path.iterdir() if p.is_dir()) + config_text = next(run_dir.glob("config_*.yaml")).read_text() + assert "power_band" in config_text + + class TestReferenceBlock: """``reference_block`` is the caller's statement of one toggle cycle: required, never inferred.""" From 8be7a7a4bd8160d1eae5c93a8f654f9e8e5cf5f2 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 15:43:36 +0100 Subject: [PATCH 15/18] ToggleSpecialistMethod: rename reference_block to toggle_period, drop segment_imbalance_warning toggle_period says what the caller states (one toggle cycle) and no longer reads like the bootstrap block or the block leg. The imbalance warning was hard to interpret; the same per-stage kept fractions stay in selection_accounting and the selection CSV. Co-Authored-By: Claude Opus 5.5 --- .../baselines/study_toggle_methods_compare.py | 2 +- .../study_toggle_specialist_uncertainty.py | 2 +- benchmarking/baselines/toggle_specialist.py | 59 +++----- benchmarking/campaigns/methods.py | 2 +- .../baselines/test_toggle_specialist.py | 140 ++++++++---------- 5 files changed, 85 insertions(+), 120 deletions(-) diff --git a/benchmarking/baselines/study_toggle_methods_compare.py b/benchmarking/baselines/study_toggle_methods_compare.py index f90fd6fe..df32768b 100644 --- a/benchmarking/baselines/study_toggle_methods_compare.py +++ b/benchmarking/baselines/study_toggle_methods_compare.py @@ -202,7 +202,7 @@ def _build_methods(out_dir: Path, *, era5_hourly_df: pd.DataFrame) -> list: return [ ToggleSpecialistMethod( columns=HOT_COLUMNS, - reference_block=DEFAULT_TOGGLE_PERIOD, + toggle_period=DEFAULT_TOGGLE_PERIOD, conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, out_dir=out_dir / "toggle_specialist_runs", diff --git a/benchmarking/baselines/study_toggle_specialist_uncertainty.py b/benchmarking/baselines/study_toggle_specialist_uncertainty.py index b78a6ed1..b3869b82 100644 --- a/benchmarking/baselines/study_toggle_specialist_uncertainty.py +++ b/benchmarking/baselines/study_toggle_specialist_uncertainty.py @@ -129,7 +129,7 @@ def build_methods(block_hours_grid: list[float], out_dir: Path | None = None) -> return [ ToggleSpecialistMethod( columns=HOT_COLUMNS, - reference_block=DEFAULT_TOGGLE_PERIOD, + toggle_period=DEFAULT_TOGGLE_PERIOD, name=f"toggle_specialist_bl{block_hours:g}", conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index d6c9f313..81669f26 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -16,9 +16,9 @@ other segment nearby, so a filter that removes one segment's rows in some conditions also removes the other segment's rows from those conditions. Rows kept per segment after each selection stage are reported as ``MethodOutput.selection_accounting`` and written to a selection CSV. A caller may also -flag rows through ``columns.exclude_block``: such a row removes its whole reference block (one -toggle cycle), so both states lose the same conditions — a symmetric alternative to ``exclude_row`` -for a selection that would otherwise thin one state's rows in particular conditions. +flag rows through ``columns.exclude_block``: such a row removes its whole toggle cycle, so both +states lose the same conditions — a symmetric alternative to ``exclude_row`` for a selection that +would otherwise thin one state's rows in particular conditions. **Two legs, blended.** The test power is compared against two references, each estimated on its own row selection, and the two estimates are blended by minimum-variance weights, headline and @@ -27,9 +27,9 @@ subset whose noise is smallest (ties to the larger subset), so a reference that tracks badly, or is screened out for long stretches, loses to the subsets without it; every subset's result is written to a CSV. The *block leg* (``block_mean``) needs **no references at all**: the campaign is tiled -into fixed wall-clock blocks of ``reference_block`` (one toggle cycle) and every used row -of a block is compared against the block's mean test power, so the on/off ratio cancels the block's -wind level exactly; a block whose used rows lack either state is dropped (the ``block`` selection +into fixed wall-clock toggle cycles of ``toggle_period`` and every used row of a cycle is +compared against the cycle's mean test power, so the on/off ratio cancels the cycle's wind level +exactly; a cycle whose used rows lack either state is dropped (the ``block`` selection stage). The two legs' errors come from different places (reference mismatch versus within-cycle drift), which is what the blend exploits; their bootstraps share their block draws, so the blend's sigma includes the legs' correlation (:func:`benchmarking.baselines.block_bootstrap.combine_estimates`). @@ -37,7 +37,7 @@ **Decisions on permuted labels.** The two data-driven decisions, the subset choice and the blend weights, are judged on a bootstrap of the same rows with the on/off labels shuffled within each -reference block: the same noise, blind to how it happened to split between the states. Ranking on +toggle cycle: the same noise, blind to how it happened to split between the states. Ranking on the realised sigma would favour the alternative whose realised uplift came out low (the on/off contrast noise is skewed), biasing the estimate down and under-reporting sigma. The reported uplift and sigma are always the actual ones; only the decisions read the permuted bootstrap. @@ -307,10 +307,8 @@ class ToggleSpecialistMethod: :param pairing_max_gap: when set, a used row of one segment is kept only if the other segment has a used row within this gap, inclusive (a gap of exactly ``pairing_max_gap`` is kept). Must be a positive whole multiple of the timebase; ``None`` disables pairing. - :param segment_imbalance_warning: warn when the two segments' kept fractions at any selection - stage differ by more than this. - :param reference_block: **required** block length for the block leg, the label permutation and - ``columns.exclude_block``: one toggle cycle (the campaign's own on + off duration), a + :param toggle_period: **required** one toggle cycle (the campaign's own on + off duration): the + block length for the block leg, the label permutation and ``columns.exclude_block``. A positive whole multiple of the timebase. It is the caller's to state, not inferred, and it is independent of ``pairing_max_gap``. :param power_band: when set, ``(lo, hi]`` in kW: each leg keeps only the used rows whose @@ -319,7 +317,7 @@ class ToggleSpecialistMethod: """ columns: ColumnSchema - reference_block: pd.Timedelta + toggle_period: pd.Timedelta name: str = "toggle_specialist" out_dir: Path | None = None save_plots: bool = False @@ -330,7 +328,6 @@ class ToggleSpecialistMethod: n_resamples: int = 1000 bootstrap_seed: int = 0 pairing_max_gap: pd.Timedelta | None = None - segment_imbalance_warning: float = 0.1 power_band: tuple[float, float] | None = None def __post_init__(self) -> None: @@ -366,7 +363,7 @@ def estimate(self, mi: MethodInput) -> MethodOutput: timebase = self.timebase if self.timebase is not None else _infer_timebase(mi.scada_df.index) self._check_pairing_gap(timebase) rows = resolve_toggle(mi.upgrade_timing, wide_all.index) - block = self._reference_block(timebase) + block = self._toggle_period(timebase) components: dict[str, _Estimate] = {} subsets: pd.DataFrame | None = None @@ -763,14 +760,14 @@ def _check_pairing_gap(self, timebase: pd.Timedelta) -> None: ) raise ValueError(msg) - def _reference_block(self, timebase: pd.Timedelta) -> pd.Timedelta: - """Return ``reference_block`` after checking it tiles the timebase grid.""" - block = self.reference_block + def _toggle_period(self, timebase: pd.Timedelta) -> pd.Timedelta: + """Return ``toggle_period`` after checking it tiles the timebase grid.""" + block = self.toggle_period ratio = block / timebase if block <= pd.Timedelta(0) or ratio != round(ratio): msg = ( - f"{self.name}: reference_block {block} must be a positive whole multiple of the timebase " - f"{timebase}; a block that does not tile the timebase grid cannot hold whole records." + f"{self.name}: toggle_period {block} must be a positive whole multiple of the timebase " + f"{timebase}; a period that does not tile the timebase grid cannot hold whole records." ) raise ValueError(msg) return block @@ -813,7 +810,7 @@ def _selection( ) -> _Selection: """Return the used-row mask after each selection stage, each a bool Series on ``wide.index``. - ``ids`` are the reference-block ids (``None`` when there are no campaign rows to tile). The + ``ids`` are the toggle-cycle ids (``None`` when there are no campaign rows to tile). The ``block`` stage drops every row of a block that carries a ``columns.exclude_block`` flag on any of the turbines (both legs) or, when ``both_states`` (the block leg), lacks either state. """ @@ -853,7 +850,7 @@ def _selection( def _selection_accounting( self, selection: _Selection, *, rows: ToggleRowSets, power_band: npt.NDArray[np.bool_] | None = None ) -> pd.DataFrame: - """Rows kept per segment after each stage, warning when a stage treats the segments unevenly. + """Rows kept per segment after each stage. ``kept_fraction`` is relative to the segment's rows; ``stage_kept_fraction`` to the previous stage, so it isolates which stage discriminates between segments. @@ -882,23 +879,7 @@ def _selection_accounting( } ) previous = n_kept - accounting = pd.DataFrame(records) - - by_stage = accounting.pivot( # noqa: PD010 - one row per (stage, segment), nothing to aggregate - index="stage", columns="segment", values="stage_kept_fraction" - ) - imbalance = (by_stage[_BASELINE] - by_stage[_UPGRADED]).abs() - for stage, gap in imbalance[imbalance > self.segment_imbalance_warning].items(): - logger.warning( - "%s: stage %r kept %.1f%% of baseline vs %.1f%% of upgraded rows (difference %.1f%% > %.1f%%)", - self.name, - stage, - 100 * by_stage.loc[stage, _BASELINE], - 100 * by_stage.loc[stage, _UPGRADED], - 100 * gap, - 100 * self.segment_imbalance_warning, - ) - return accounting + return pd.DataFrame(records) def _filtered_mask( self, mi: MethodInput, *, wide: pd.DataFrame, test: str, refs: list[str], timebase: pd.Timedelta @@ -1083,7 +1064,7 @@ def _write_shared_diagnostics( "active_power_col": self.columns.active_power, "availability_col": self.columns.availability, "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), - "reference_block": str(self.reference_block), + "toggle_period": str(self.toggle_period), "power_band": None if self.power_band is None else list(self.power_band), "exclude_block_col": self.columns.exclude_block, } diff --git a/benchmarking/campaigns/methods.py b/benchmarking/campaigns/methods.py index 4a1216e8..e721b3ba 100644 --- a/benchmarking/campaigns/methods.py +++ b/benchmarking/campaigns/methods.py @@ -56,7 +56,7 @@ def carried_forward_methods( methods.append( ToggleSpecialistMethod( columns=HOT_COLUMNS, - reference_block=spec.upgrade_timing.period, + toggle_period=spec.upgrade_timing.period, out_dir=out_dir / "toggle_specialist", save_plots=True, conditions=("power",), diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 474e8263..26e6e82c 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -43,7 +43,7 @@ # Larger than any test timebase's full period, so the (required) availability filter keeps # every row unless a test deliberately sets a lower value. _FULLY_AVAILABLE_SECS = 3600.0 -# One toggle cycle of the 20-minute ``ToggleSchedule`` most tests use (``reference_block`` is required). +# One toggle cycle of the 20-minute ``ToggleSchedule`` most tests use (``toggle_period`` is required). _CYCLE = pd.Timedelta(minutes=20) # The method reads active_power + availability from the schema; the other required roles are # unused by the ratio, so name them with placeholders. @@ -121,7 +121,7 @@ def test_toggle_specialist_shares_no_wind_up_code() -> None: class TestRecovery: def test_toggle_recovers_known_uplift(self) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert isinstance(out, MethodOutput) @@ -174,7 +174,7 @@ def _varying_rho_case( def _per_bin(scada: pd.DataFrame, schedule: ToggleSchedule) -> pd.DataFrame: """Run the method with power conditioning on and return its populated per-bin rows.""" out = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None frame = out.p50_by_condition @@ -249,7 +249,7 @@ class TestConditionsConfiguration: def test_default_reports_no_conditions(self) -> None: # back-compat: existing callers get exactly today's behaviour and need no rating scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_by_condition is None @@ -258,16 +258,16 @@ def test_power_conditioning_does_not_move_the_headline(self) -> None: # the per-bin decomposition is additional reporting, never a change to the estimate scada, schedule, _ = _toggle_case(uplift=0.03) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - plain = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate(mi) + plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate(mi) conditioned = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(mi) assert conditioned.p50_overall == plain.p50_overall def test_frame_is_labelled_with_the_power_condition(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert set(out.p50_by_condition["condition"]) == {"power"} @@ -276,19 +276,17 @@ def test_frame_is_labelled_with_the_power_condition(self) -> None: def test_ws_condition_raises_citing_the_method_limit(self) -> None: with pytest.raises(ValueError, match="does not support"): - ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("ws",), rated_power_kw=_RATED_KW - ) + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, conditions=("ws",), rated_power_kw=_RATED_KW) def test_unknown_condition_raises(self) -> None: with pytest.raises(ValueError, match="unknown condition"): ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("bogus",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("bogus",), rated_power_kw=_RATED_KW ) def test_power_without_a_rating_raises(self) -> None: with pytest.raises(ValueError, match="rated_power_kw"): - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",)) + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",)) class TestPerBinSparseData: @@ -298,7 +296,7 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: # above the data's power range guarantees empty upper bins rather than hoping for them. scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=4000.0 + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=4000.0 ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None empty = out.p50_by_condition[out.p50_by_condition["n_records"] == 0] @@ -308,7 +306,7 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: def test_every_bin_is_represented(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert len(out.p50_by_condition) == len(condition_bins("power", rated_power_kw=_RATED_KW)) - 1 @@ -318,13 +316,13 @@ class TestPerBinDiagnostics: def test_per_bin_csv_is_written(self, tmp_path: Path) -> None: scada, schedule = _varying_rho_case(uplift=0.05) ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert list(tmp_path.rglob("*_by_power_bin_*.csv")) def test_no_per_bin_csv_when_conditioning_is_off(self, tmp_path: Path) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert not list(tmp_path.rglob("*_by_power_bin_*.csv")) @@ -333,7 +331,7 @@ def test_per_bin_plot_written_with_save_plots(self, tmp_path: Path) -> None: scada, schedule = _varying_rho_case(uplift=0.05) ToggleSpecialistMethod( columns=_COLUMNS, - reference_block=_CYCLE, + toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path, @@ -351,7 +349,7 @@ def test_prepost_timestamp_raises(self) -> None: treated = np.asarray(idx >= upgrade) scada = _recovery_scada(idx, treated=treated, uplift=0.05) with pytest.raises(ValueError, match="toggle"): - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=upgrade, turbine_col=_TURBINE_COL) ) @@ -368,13 +366,13 @@ def test_blank_availability_role_raises(self, blank: str) -> None: # A schema that leaves the availability role blank (empty or whitespace) would silently skip # downtime filtering, so construction must reject it. with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=replace(_COLUMNS, availability=blank), reference_block=_CYCLE) + ToggleSpecialistMethod(columns=replace(_COLUMNS, availability=blank), toggle_period=_CYCLE) def test_missing_availability_column_raises(self) -> None: scada, schedule, _ = _toggle_case() scada = scada.drop(columns=[_AVAIL_COL]) with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -387,7 +385,7 @@ def test_down_reference_timestamps_are_excluded(self) -> None: down = (corrupted[_TURBINE_COL] == "R1") & corrupted.index.isin([idx[3], idx[4]]) corrupted.loc[down, _POWER_COL] = 1e6 corrupted.loc[down, _AVAIL_COL] = 0.0 - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(0.05) @@ -397,7 +395,7 @@ class TestCompleteCase: def test_drops_timestamp_when_a_reference_is_nan(self) -> None: scada, schedule, _ = _toggle_case() idx = scada.index.unique().sort_values() - clean = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + clean = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -406,7 +404,7 @@ def test_drops_timestamp_when_a_reference_is_nan(self) -> None: corrupted = scada.copy() mask = (corrupted[_TURBINE_COL] == "R1") & (corrupted.index == idx[3]) corrupted.loc[mask, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(clean.p50_overall) @@ -417,12 +415,12 @@ def test_nan_at_unused_timestamp_does_not_change_estimate(self) -> None: # make idx[2] already unused (test NaN), then add a second NaN at the same timestamp base = scada.copy() base.loc[(base[_TURBINE_COL] == "T1") & (base.index == idx[2]), _POWER_COL] = np.nan - before = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + before = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=base, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) after_df = base.copy() after_df.loc[(after_df[_TURBINE_COL] == "R1") & (after_df.index == idx[2]), _POWER_COL] = np.nan - after = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + after = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=after_df, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert after.p50_overall == pytest.approx(before.p50_overall) @@ -437,7 +435,7 @@ def test_estimate_invariant_to_timebase(self) -> None: treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.04) results.append( - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=schedule.period) + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=schedule.period) .estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) .p50_overall ) @@ -447,7 +445,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 scada, schedule, _ = _toggle_case() method = ToggleSpecialistMethod( columns=_COLUMNS, - reference_block=pd.Timedelta(hours=1), + toggle_period=pd.Timedelta(hours=1), out_dir=tmp_path, timebase=pd.Timedelta(minutes=30), ) @@ -462,7 +460,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 class TestActivePowerOnly: def test_extra_columns_ignored(self) -> None: scada, schedule, _ = _toggle_case() - plain = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -470,7 +468,7 @@ def test_extra_columns_ignored(self) -> None: rng = np.random.default_rng(0) with_extra["ws"] = rng.normal(size=len(with_extra)) with_extra["rpm"] = rng.normal(size=len(with_extra)) - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=with_extra, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(plain.p50_overall) @@ -482,7 +480,7 @@ def test_no_reference_is_the_block_leg_alone(self, caplog: pytest.LogCaptureFixt schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=idx[0]) scada = _scada({"T1": np.ones(len(idx))}, idx) with caplog.at_level("WARNING"): - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert _legs(out) == {"block_mean"} @@ -495,7 +493,7 @@ def test_no_used_baseline_returns_nan(self) -> None: off_ts = idx[~treated] is_baseline_test = (scada[_TURBINE_COL] == "T1") & scada.index.isin(off_ts) scada.loc[is_baseline_test, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert np.isnan(out.p50_overall) @@ -548,9 +546,7 @@ def _read_only_csv(folder: Path, kind: str) -> pd.DataFrame: class TestDiagnostics: def _run(self, tmp_path, *, save_plots: bool = False): # noqa: ANN001, ANN202 scada, schedule, _ = _toggle_case(uplift=0.06) - method = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=save_plots - ) + method = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=save_plots) out = method.estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -609,7 +605,7 @@ def test_precampaign_dropped_from_baseline(self, tmp_path) -> None: # noqa: ANN schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=start) treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.05) - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) baseline = _read_only_csv(tmp_path, "data_stats").set_index("segment").loc["baseline"] @@ -646,7 +642,7 @@ def _long_diagnostics(out: MethodOutput) -> pd.DataFrame: def _estimate(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: - kwargs.setdefault("reference_block", _CYCLE) + kwargs.setdefault("toggle_period", _CYCLE) return ToggleSpecialistMethod(columns=_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -737,7 +733,7 @@ def test_a_nan_uplift_reports_a_nan_sigma(self) -> None: idx = _index(40) # every row treated -> no off rows -> rho_base is NaN scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput( scada_df=scada, test_wtg="T1", @@ -754,7 +750,7 @@ def test_the_bootstrap_is_not_run_when_the_uplift_is_nan(self) -> None: idx = _index(40) scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) out = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ).estimate( MethodInput( scada_df=scada, @@ -843,7 +839,7 @@ def test_one_row_per_test_turbine_record(self) -> None: def test_used_reproduces_the_methods_own_used_mask(self) -> None: scada, schedule = _noisy_toggle_case() method = ToggleSpecialistMethod( - columns=_COLUMNS, reference_block=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW ) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) out = method.estimate(mi) @@ -1006,7 +1002,7 @@ class TestExcludeRow: def test_flagged_test_rows_are_not_used(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1022,10 +1018,10 @@ def test_excluding_corrupted_rows_restores_the_known_uplift(self) -> None: spoiled = corrupted[_EXCLUDE_COL].to_numpy(dtype=bool) & (corrupted[_TURBINE_COL] == "T1").to_numpy() corrupted.loc[spoiled, _POWER_COL] *= 0.5 - naive = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + naive = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( + filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert filtered.p50_overall == pytest.approx(0.05, abs=1e-9) @@ -1042,7 +1038,7 @@ def test_flagged_reference_rows_leave_the_reference_leg_like_an_outage(self, tmp flagged[_EXCLUDE_COL] = ( flagged[_EXCLUDE_COL].to_numpy() | _flag(scada, turbine="R2", every=3)[_EXCLUDE_COL].to_numpy() ) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1060,14 +1056,14 @@ def test_nan_in_the_exclude_column_is_rejected(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=7, value=np.nan) with pytest.raises(ValueError, match=_EXCLUDE_COL): - ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) def test_unset_role_excludes_nothing(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1077,7 +1073,7 @@ def test_unset_role_excludes_nothing(self) -> None: def test_absent_column_excludes_nothing(self) -> None: """The role names a column the frame does not carry: skip, do not raise.""" scada, schedule = _noisy_toggle_case() - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1093,7 +1089,7 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) ToggleSpecialistMethod( - columns=_EXCLUDE_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=True + columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=True ).estimate(MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) names = {p.name for p in (run_dir / "plots").rglob("*.png")} @@ -1102,7 +1098,7 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: def test_no_exclusion_plots_when_nothing_is_excluded(self, tmp_path: Path) -> None: scada, schedule = _noisy_toggle_case() - ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE, out_dir=tmp_path, save_plots=True).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=True).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) @@ -1138,7 +1134,7 @@ def _fixture() -> tuple[pd.DataFrame, ToggleSchedule]: @staticmethod def _estimate(scada: pd.DataFrame, **kwargs: object) -> float: - method = ToggleSpecialistMethod(columns=_COLUMNS, reference_block=_CYCLE) + method = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE) return method.estimate(MethodInput(scada_df=scada, **kwargs)).p50_overall def test_only_offered_references_are_used(self) -> None: @@ -1262,7 +1258,7 @@ def _block_excluded_case() -> tuple[pd.DataFrame, ToggleSchedule, pd.DatetimeInd @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: - kwargs.setdefault("reference_block", _CYCLE) + kwargs.setdefault("toggle_period", _CYCLE) return ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -1308,18 +1304,6 @@ def test_accounting_is_reported_without_pairing(self) -> None: kept = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] assert kept["pairing", "baseline"] == kept["exclude_row", "baseline"] - def test_an_uneven_stage_warns(self, caplog: pytest.LogCaptureFixture) -> None: - scada, schedule, _ = self._block_excluded_case() - with caplog.at_level("WARNING", logger=toggle_specialist.__name__): - self._run(scada, schedule, segment_imbalance_warning=0.05) - assert any("exclude_row" in record.getMessage() for record in caplog.records) - - def test_an_even_selection_does_not_warn(self, caplog: pytest.LogCaptureFixture) -> None: - scada, schedule = _noisy_toggle_case(n=200) - with caplog.at_level("WARNING", logger=toggle_specialist.__name__): - _estimate(scada, schedule, pairing_max_gap=pd.Timedelta(minutes=20)) - assert not [r for r in caplog.records if r.name == toggle_specialist.__name__] - @pytest.mark.parametrize("gap", [None, pd.Timedelta(minutes=20)], ids=["no-pairing", "pairing"]) def test_pairing_gap_plot_is_written(tmp_path: Path, gap: pd.Timedelta | None) -> None: @@ -1414,22 +1398,22 @@ def test_the_band_is_a_selection_stage_and_in_the_run_config(self, tmp_path: Pat assert "power_band" in config_text -class TestReferenceBlock: - """``reference_block`` is the caller's statement of one toggle cycle: required, never inferred.""" +class TestTogglePeriod: + """``toggle_period`` is the caller's statement of one toggle cycle: required, never inferred.""" def test_it_is_required(self) -> None: - with pytest.raises(TypeError, match="reference_block"): + with pytest.raises(TypeError, match="toggle_period"): ToggleSpecialistMethod(columns=_COLUMNS) # type: ignore[call-arg] def test_it_is_independent_of_the_pairing_gap(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) - out = _estimate(scada, schedule, reference_block=_BLOCK, pairing_max_gap=pd.Timedelta(minutes=10)) + out = _estimate(scada, schedule, toggle_period=_BLOCK, pairing_max_gap=pd.Timedelta(minutes=10)) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) def test_a_block_off_the_timebase_grid_is_refused(self) -> None: scada, schedule = _block_case(n_blocks=10, ref_scale=np.ones(10)) - with pytest.raises(ValueError, match="reference_block"): - _estimate(scada, schedule, reference_block=pd.Timedelta(minutes=25)) + with pytest.raises(ValueError, match="toggle_period"): + _estimate(scada, schedule, toggle_period=pd.Timedelta(minutes=25)) class TestBlockLeg: @@ -1437,7 +1421,7 @@ class TestBlockLeg: def test_recovers_the_uplift_with_no_references(self) -> None: scada, schedule = _block_case(uplift=0.04) - out = _estimate(scada, schedule, reference_block=_BLOCK) + out = _estimate(scada, schedule, toggle_period=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) assert np.isfinite(out.sigma_overall) assert _legs(out) == {"block_mean"} @@ -1447,7 +1431,7 @@ def test_ignores_a_reference_outage(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() scada.loc[is_ref, _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_block=_BLOCK) + out = _estimate(scada, schedule, toggle_period=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) assert out.labeled_rows is not None assert out.labeled_rows["used"].astype(bool).all() @@ -1459,7 +1443,7 @@ def test_a_block_with_one_state_is_dropped_and_accounted(self) -> None: block_rows = idx[40:44] # the 11th block: two on/off cycles off_rows = block_rows[~np.asarray(treated_mask(block_rows, schedule))] scada.loc[scada.index.isin(off_rows), _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_block=_BLOCK) + out = _estimate(scada, schedule, toggle_period=_BLOCK) assert out.labeled_rows is not None # NaN (a timestamp absent from the pivot) must read as unused, so compare rather than cast used = out.labeled_rows["used"] == True # noqa: E712 @@ -1483,7 +1467,7 @@ class TestBlend: def test_recovers_the_uplift_when_both_legs_are_exact(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) - out = _estimate(scada, schedule, reference_block=_BLOCK) + out = _estimate(scada, schedule, toggle_period=_BLOCK) assert out.p50_overall == pytest.approx(0.04, abs=1e-9) assert _legs(out) == {"sum", "block_mean", "combined"} @@ -1491,7 +1475,7 @@ def test_used_rows_are_the_union_of_the_legs(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) is_ref = (scada[_TURBINE_COL] == "R1").to_numpy() scada.loc[is_ref & (np.arange(len(scada)) % 7 == 0), _POWER_COL] = np.nan - out = _estimate(scada, schedule, reference_block=_BLOCK) + out = _estimate(scada, schedule, toggle_period=_BLOCK) used = out.labeled_rows["used"] == True # noqa: E712 - NaN (absent timestamp) must read as unused assert used.all() # The accounting is the reference leg's, which the reference outage did cost rows. @@ -1547,7 +1531,7 @@ def _flag_block(scada: pd.DataFrame, *, turbine: str, at: pd.Timestamp) -> pd.Da class TestExcludeBlock: - """``columns.exclude_block``: a flagged row removes its whole reference block, in both legs.""" + """``columns.exclude_block``: a flagged row removes its whole toggle cycle, in both legs.""" @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: @@ -1559,7 +1543,7 @@ def test_a_flagged_test_row_removes_its_whole_block(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) idx = pd.DatetimeIndex(pd.unique(scada.index)) block_5 = idx[20:24] - out = self._run(_flag_block(scada, turbine="T1", at=block_5[1]), schedule, reference_block=_BLOCK) + out = self._run(_flag_block(scada, turbine="T1", at=block_5[1]), schedule, toggle_period=_BLOCK) assert out.labeled_rows is not None used = out.labeled_rows["used"].astype(bool) assert not used.loc[block_5].any() @@ -1574,7 +1558,7 @@ def test_a_flagged_reference_row_removes_the_block_from_the_reference_leg(self) scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) idx = pd.DatetimeIndex(pd.unique(scada.index)) block_7 = idx[28:32] - out = self._run(_flag_block(scada, turbine="R1", at=block_7[3]), schedule, reference_block=_BLOCK) + out = self._run(_flag_block(scada, turbine="R1", at=block_7[3]), schedule, toggle_period=_BLOCK) assert out.labeled_rows is not None assert out.labeled_rows["used"].astype(bool).all() acc = out.selection_accounting.set_index(["stage", "segment"])["n_kept"] @@ -1585,8 +1569,8 @@ def test_a_flagged_reference_row_removes_the_block_from_the_reference_leg(self) def test_no_flags_change_nothing(self) -> None: scada, schedule = _block_case(uplift=0.04, ref_scale=np.full(120, 0.5)) scada[_BLOCK_COL] = False - with_role = self._run(scada, schedule, reference_block=_BLOCK) - without = _estimate(scada.drop(columns=_BLOCK_COL), schedule, reference_block=_BLOCK) + with_role = self._run(scada, schedule, toggle_period=_BLOCK) + without = _estimate(scada.drop(columns=_BLOCK_COL), schedule, toggle_period=_BLOCK) assert with_role.p50_overall == pytest.approx(without.p50_overall) assert with_role.sigma_overall == pytest.approx(without.sigma_overall) From 16c39efc7026f319e8df55927d0e369c809f88ae Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 16:58:10 +0100 Subject: [PATCH 16/18] ToggleSpecialistMethod: explicit bin power and toggle datum, per-leg diagnostics - bin_power: "baseline" (default) bins and power_band read power no upgraded row enters (rho_base * ref_total on the reference leg, the cycle's baseline-row mean on the block leg); "both_states" keeps the two-state mean, for campaigns with no true baseline state. - toggle_datum (required): the toggle-cycle grid is tiled from it, never from the first record, so a missing record cannot shift cycle boundaries. Toggle studies state one and draw treatment starts only on cycle boundaries. - Data-stats rows and results-CSV ratios per leg, plus the blend weight, so each leg and the headline are re-derivable; plots show one leg's own rows. - Refuse more than 8 references for the subset choice; validate power_band. - ColumnSchema.exclude_row: which turbines' flags count is the method's. Co-Authored-By: Claude Opus 5.5 --- .../baselines/example_toggle_study.py | 3 + benchmarking/baselines/old/inspect_naive.py | 1 + .../baselines/study_overnight_toggle.py | 2 + .../baselines/study_power_model_compare.py | 3 +- .../baselines/study_toggle_methods_compare.py | 4 +- .../study_toggle_specialist_uncertainty.py | 4 +- benchmarking/baselines/toggle_specialist.py | 209 ++++++++++++----- benchmarking/campaigns/methods.py | 2 + benchmarking/harness/replicates.py | 35 ++- benchmarking/synthetic/schema.py | 8 +- .../baselines/test_toggle_end_to_end.py | 1 + .../baselines/test_toggle_specialist.py | 218 +++++++++++++----- tests/benchmarking/harness/test_replicates.py | 28 +++ tests/benchmarking/harness/test_scoring.py | 1 + 14 files changed, 385 insertions(+), 134 deletions(-) diff --git a/benchmarking/baselines/example_toggle_study.py b/benchmarking/baselines/example_toggle_study.py index 7990b118..955797ca 100644 --- a/benchmarking/baselines/example_toggle_study.py +++ b/benchmarking/baselines/example_toggle_study.py @@ -51,6 +51,8 @@ # 20 minutes on, 20 minutes off -> a 40-minute on/off cycle. DEFAULT_TOGGLE_PERIOD = pd.Timedelta(minutes=40) +# Cycles are counted from the Unix epoch, as a clock-aligned controller counts them. +DEFAULT_TOGGLE_DATUM = pd.Timestamp("1970-01-01", tz="UTC") # One upgrade for the toggle study: a flat +3% Cp change in region 2. TOGGLE_PROFILES: dict[str, list] = {"cp_plus_3pct": [ConstantCpChange(delta=0.03)]} @@ -199,6 +201,7 @@ def main( min_pre_months=MIN_PRE_MONTHS, campaign_months=campaign_months, toggle_period=toggle_period, + toggle_datum=DEFAULT_TOGGLE_DATUM, n_replicates=n_replicates, seed=0, ) diff --git a/benchmarking/baselines/old/inspect_naive.py b/benchmarking/baselines/old/inspect_naive.py index 9bad69c2..822321b8 100644 --- a/benchmarking/baselines/old/inspect_naive.py +++ b/benchmarking/baselines/old/inspect_naive.py @@ -171,6 +171,7 @@ def inspect_naive_run( campaign_months=campaign_months, n_replicates=n_replicates, toggle_period=toggle_period, + toggle_datum=pd.Timestamp("1970-01-01", tz="UTC"), seed=0, ) diff --git a/benchmarking/baselines/study_overnight_toggle.py b/benchmarking/baselines/study_overnight_toggle.py index cd5e9217..83918a43 100644 --- a/benchmarking/baselines/study_overnight_toggle.py +++ b/benchmarking/baselines/study_overnight_toggle.py @@ -26,6 +26,7 @@ MIN_PRE_MONTHS, ) from benchmarking.baselines.example_toggle_study import ( + DEFAULT_TOGGLE_DATUM, DEFAULT_TOGGLE_PERIOD, default_output_root, run_toggle_study, @@ -52,6 +53,7 @@ def main() -> None: min_pre_months=MIN_PRE_MONTHS, campaign_months=CAMPAIGN_MONTHS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, n_replicates=N_REPLICATES, seed=0, ) diff --git a/benchmarking/baselines/study_power_model_compare.py b/benchmarking/baselines/study_power_model_compare.py index c90fef78..270b670a 100644 --- a/benchmarking/baselines/study_power_model_compare.py +++ b/benchmarking/baselines/study_power_model_compare.py @@ -93,7 +93,7 @@ MIN_PRE_MONTHS, save_per_method_curve, ) -from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_PERIOD +from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_DATUM, DEFAULT_TOGGLE_PERIOD from benchmarking.baselines.hot_context import build_hot_v0_context from benchmarking.baselines.naive_ratio import NaiveRatioMethod from benchmarking.baselines.overnight_profiles import overnight_profiles @@ -171,6 +171,7 @@ def _toggle_study() -> StudyConfig: min_pre_months=MIN_PRE_MONTHS, campaign_months=TOGGLE_CAMPAIGN_MONTHS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, n_replicates=N_REPLICATES, seed=SEED, ) diff --git a/benchmarking/baselines/study_toggle_methods_compare.py b/benchmarking/baselines/study_toggle_methods_compare.py index df32768b..43e31523 100644 --- a/benchmarking/baselines/study_toggle_methods_compare.py +++ b/benchmarking/baselines/study_toggle_methods_compare.py @@ -61,7 +61,7 @@ DEFAULT_WTG_NUMBERS, MIN_PRE_MONTHS, ) -from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_PERIOD +from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_DATUM, DEFAULT_TOGGLE_PERIOD from benchmarking.baselines.hot_context import build_hot_v0_context from benchmarking.baselines.power_model import PowerModelMethod from benchmarking.baselines.toggle_specialist import ToggleSpecialistMethod @@ -171,6 +171,7 @@ def toggle_study() -> StudyConfig: min_pre_months=MIN_PRE_MONTHS, campaign_weeks=CAMPAIGN_WEEKS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, n_replicates=N_REPLICATES, seed=SEED, ) @@ -203,6 +204,7 @@ def _build_methods(out_dir: Path, *, era5_hourly_df: pd.DataFrame) -> list: ToggleSpecialistMethod( columns=HOT_COLUMNS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, out_dir=out_dir / "toggle_specialist_runs", diff --git a/benchmarking/baselines/study_toggle_specialist_uncertainty.py b/benchmarking/baselines/study_toggle_specialist_uncertainty.py index b3869b82..e2070fc2 100644 --- a/benchmarking/baselines/study_toggle_specialist_uncertainty.py +++ b/benchmarking/baselines/study_toggle_specialist_uncertainty.py @@ -49,7 +49,7 @@ DEFAULT_TURBINE_SUBSET, DEFAULT_WTG_NUMBERS, ) -from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_PERIOD +from benchmarking.baselines.example_toggle_study import DEFAULT_TOGGLE_DATUM, DEFAULT_TOGGLE_PERIOD from benchmarking.baselines.study_toggle_methods_compare import TOGGLE_PROFILES from benchmarking.baselines.toggle_specialist import DEFAULT_BLOCK_HOURS, ToggleSpecialistMethod from benchmarking.harness import ( @@ -99,6 +99,7 @@ def uncertainty_study(n_replicates: int) -> StudyConfig: min_pre_months=MIN_PRE_MONTHS_TOGGLE, campaign_weeks=CAMPAIGN_WEEKS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, n_replicates=n_replicates, seed=SEED, ) @@ -130,6 +131,7 @@ def build_methods(block_hours_grid: list[float], out_dir: Path | None = None) -> ToggleSpecialistMethod( columns=HOT_COLUMNS, toggle_period=DEFAULT_TOGGLE_PERIOD, + toggle_datum=DEFAULT_TOGGLE_DATUM, name=f"toggle_specialist_bl{block_hours:g}", conditions=("power",), rated_power_kw=HOT_RATED_POWER_KW, diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 81669f26..c125ad95 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -51,10 +51,18 @@ Each run writes a per-run folder ``toggle_specialist___/`` (v0-style naming) under ``out_dir`` (a temp dir by default), holding a per-segment data-stats CSV, a headline results CSV, the per-leg selection and reference-subset CSVs, and -- when -``save_plots`` -- three diagnostic plots (a test-vs-reference scatter, a per-segment daily-ratio -timeseries, and a per-segment used-data-coverage timeseries). -The rich stats let a human confirm the right data was received and interpreted: the headline uplift -is re-derivable from the stats CSV as ``rho = used_test_mwh / used_ref_total_mwh`` per segment. +``save_plots`` -- four diagnostic plots (a test-vs-reference scatter, a per-segment daily-ratio +timeseries, a per-segment used-data-coverage timeseries, and a pairing-gap histogram). +The rich stats let a human confirm the right data was received and interpreted: each leg's uplift +is re-derivable from its rows of the stats CSV as ``rho = used_test_mwh / used_ref_total_mwh`` per +segment, and the headline is the blend of the legs at the weight the results CSV reports. + +**Bin power.** The power bins and ``power_band`` both read one per-row power, chosen by +``bin_power``. ``"baseline"`` (the default) is what the turbine makes untreated: ``rho_base * +ref_total`` on the reference leg, the mean of the cycle's baseline rows on the block leg; no +upgraded row enters it. ``"both_states"`` is the mean of the two states instead (``rho`` averaged +over the states; the two states' cycle means averaged), for a campaign in which neither state is +a true baseline. Either way a row's own state cannot move it between bins. """ from __future__ import annotations @@ -64,7 +72,7 @@ from dataclasses import dataclass, replace from itertools import combinations from pathlib import Path -from typing import TYPE_CHECKING, NamedTuple +from typing import TYPE_CHECKING, Literal, NamedTuple import matplotlib.pyplot as plt import numpy as np @@ -111,8 +119,9 @@ _REFERENCE_LEG = "sum" _BLOCK_LEG = "block_mean" _COMBINED = "combined" -# The subset choice runs 2**n - 1 estimates; warn, rather than refuse, past this many references. -_MAX_REFS_FOR_SUBSETS = 6 +# The subset choice runs 2**n - 1 estimates; refuse past this many references. +_MAX_REFS_FOR_SUBSETS = 8 +_BIN_POWERS = ("baseline", "both_states") logger = logging.getLogger(__name__) @@ -311,13 +320,19 @@ class ToggleSpecialistMethod: block length for the block leg, the label permutation and ``columns.exclude_block``. A positive whole multiple of the timebase. It is the caller's to state, not inferred, and it is independent of ``pairing_max_gap``. - :param power_band: when set, ``(lo, hi]`` in kW: each leg keeps only the used rows whose - reference-derived baseline power (``rho_label * ref_total``, the power the bins use, so - state-neutral) lies in it, and estimates its uplift over those. ``None`` keeps every used row. + :param toggle_datum: **required** any boundary of the toggle-cycle grid (the controller's own + cycle datum, not when the campaign happened to begin); the cycles of the block leg, the label + permutation and ``columns.exclude_block`` are tiled from it. A ``ToggleSchedule`` tiles its + cycles from its first timestamp, which must lie on this grid. + :param bin_power: the per-row power the bins and ``power_band`` read (see the module docstring): + ``"baseline"`` (default) or ``"both_states"``. + :param power_band: when set, ``(lo, hi]`` in kW: each leg keeps only the used rows whose bin + power lies in it, and estimates its uplift over those. ``None`` keeps every used row. """ columns: ColumnSchema toggle_period: pd.Timedelta + toggle_datum: pd.Timestamp name: str = "toggle_specialist" out_dir: Path | None = None save_plots: bool = False @@ -328,12 +343,21 @@ class ToggleSpecialistMethod: n_resamples: int = 1000 bootstrap_seed: int = 0 pairing_max_gap: pd.Timedelta | None = None + bin_power: Literal["baseline", "both_states"] = "baseline" power_band: tuple[float, float] | None = None def __post_init__(self) -> None: - """Validate ``columns`` names every role this method reads, and the requested ``conditions``.""" + """Validate the column roles, ``conditions``, ``bin_power`` and ``power_band``.""" self.columns.require_roles(("active_power", "availability")) validate_conditions(self.conditions, supported=_SUPPORTED_CONDITIONS, method_name=self.name) + if self.bin_power not in _BIN_POWERS: + msg = f"{self.name}: bin_power must be one of {_BIN_POWERS}, got {self.bin_power!r}" + raise ValueError(msg) + if self.power_band is not None: + lo, hi = self.power_band + if not (np.isfinite(lo) and np.isfinite(hi) and lo < hi): + msg = f"{self.name}: power_band must be finite (lo, hi] with lo < hi, got {self.power_band}" + raise ValueError(msg) if "power" in self.conditions and self.rated_power_kw is None: msg = ( f"{self.name}: rated_power_kw is required when 'power' is in conditions — the power bin " @@ -364,13 +388,14 @@ def estimate(self, mi: MethodInput) -> MethodOutput: self._check_pairing_gap(timebase) rows = resolve_toggle(mi.upgrade_timing, wide_all.index) block = self._toggle_period(timebase) + datum = self._toggle_datum(mi, wide_all.index) components: dict[str, _Estimate] = {} subsets: pd.DataFrame | None = None n_refs_available = len(mi.context.references_among(wide_all.columns)) if n_refs_available: components[_REFERENCE_LEG], subsets = self._choose_references( - mi, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + mi, wide_all=wide_all, test=test, timebase=timebase, block=block, datum=datum, rows=rows ) else: # Nothing for the reference leg to work with: the estimate is the block leg alone, which @@ -382,7 +407,14 @@ def estimate(self, mi: MethodInput) -> MethodOutput: _BLOCK_LEG, ) components[_BLOCK_LEG] = self._component( - mi, mode=_BLOCK_LEG, wide_all=wide_all, test=test, timebase=timebase, block=block, rows=rows + mi, + mode=_BLOCK_LEG, + wide_all=wide_all, + test=test, + timebase=timebase, + block=block, + datum=datum, + rows=rows, ) est = ( _combine(components[_REFERENCE_LEG], components[_BLOCK_LEG], upgraded=rows.upgraded, bins=self._bins()) @@ -392,14 +424,20 @@ def estimate(self, mi: MethodInput) -> MethodOutput: ref_set = set(est.refs) wide = wide_all[[c for c in wide_all.columns if c == test or c in ref_set]] - stats = _segment_stats( - mi, - wide=wide, - used=est.used, - toggle_rows=rows, - ref_total=est.ref_total, - timebase=timebase, - active_power_col=self.columns.active_power, + stats = pd.concat( + [ + _segment_stats( + mi, + wide=wide_all[[c for c in wide_all.columns if c == test or c in set(component.refs)]], + used=component.used, + toggle_rows=rows, + ref_total=component.ref_total, + timebase=timebase, + active_power_col=self.columns.active_power, + ).assign(component=mode) + for mode, component in components.items() + ], + ignore_index=True, ) self._write_outputs( mi, @@ -430,6 +468,7 @@ def _component( test: str, timebase: pd.Timedelta, block: pd.Timedelta | None, + datum: pd.Timestamp, rows: ToggleRowSets, refs: list[str] | None = None, with_actual: bool = True, @@ -453,11 +492,7 @@ def _component( baseline = rows.campaign_baseline test_pw = wide[test].to_numpy(dtype=float) campaign = baseline | rows.upgraded - ids = ( - block_ids(wide.index, start=wide.index[campaign].min(), block=block) - if block is not None and campaign.any() - else None - ) + ids = block_ids(wide.index, start=datum, block=block) if block is not None and campaign.any() else None selection = self._selection( mi, wide=wide, test=test, refs=refs, timebase=timebase, rows=rows, ids=ids, both_states=mode == _BLOCK_LEG ) @@ -466,11 +501,19 @@ def _component( rho_base = _rho(test_pw, ref_total, used & baseline) rho_up = _rho(test_pw, ref_total, used & rows.upgraded) - rho_label = _rho_label(rho_base, rho_up) + label = self._bin_power( + mode=mode, + test_pw=test_pw, + ref_total=ref_total, + baseline=used & baseline, + upgraded=used & rows.upgraded, + ids=ids, + rho_base=rho_base, + rho_up=rho_up, + ) in_band = None if self.power_band is not None: lo, hi = self.power_band - label = rho_label * ref_total in_band = used & (label > lo) & (label <= hi) used = in_band rho_base = _rho(test_pw, ref_total, used & baseline) @@ -483,7 +526,7 @@ def _component( self._conditional_frame( test_pw=test_pw, ref_total=ref_total, - rho_label=rho_label, + label=label, baseline=used & baseline, upgraded=used & rows.upgraded, ) @@ -493,7 +536,7 @@ def _component( # Uncertainty runs strictly after the uplift, off the same frozen row selection and bin # assignment, and only when there is a finite uplift to qualify. - membership = self._cell_membership(rho_label=rho_label, ref_total=ref_total, used=used) + membership = self._cell_membership(label=label, used=used) def _run_bootstrap(up: npt.NDArray[np.bool_], base: npt.NDArray[np.bool_]) -> BootstrapResult: return self._bootstrap( @@ -535,7 +578,7 @@ def _run_bootstrap(up: npt.NDArray[np.bool_], base: npt.NDArray[np.bool_]) -> Bo selection=selection, used=used, ref_total=ref_total, - label=rho_label * ref_total, + label=label, rho_base=rho_base, rho_up=rho_up, uplift=uplift, @@ -555,6 +598,7 @@ def _choose_references( test: str, timebase: pd.Timedelta, block: pd.Timedelta | None, + datum: pd.Timestamp, rows: ToggleRowSets, ) -> tuple[_Estimate, pd.DataFrame | None]: """Run the ``sum`` leg on every non-empty reference subset; return the smallest-sigma one. @@ -566,12 +610,11 @@ def _choose_references( """ refs_all = mi.context.references_among(wide_all.columns) if len(refs_all) > _MAX_REFS_FOR_SUBSETS: - logger.warning( - "%s: the reference-subset choice over %d references runs %d estimates", - self.name, - len(refs_all), - 2 ** len(refs_all) - 1, + msg = ( + f"{self.name}: {len(refs_all)} references would need {2 ** len(refs_all) - 1} subset estimates; " + f"pass at most {_MAX_REFS_FOR_SUBSETS} (the nearest or best-correlated) references" ) + raise ValueError(msg) subsets = [list(c) for k in range(1, len(refs_all) + 1) for c in combinations(refs_all, k)] estimates = [ self._component( @@ -581,6 +624,7 @@ def _choose_references( test=test, timebase=timebase, block=block, + datum=datum, rows=rows, refs=refs, with_actual=False, @@ -600,6 +644,7 @@ def _rank(i: int) -> tuple[float, int, int]: test=test, timebase=timebase, block=block, + datum=datum, rows=rows, refs=subsets[best], ) @@ -641,7 +686,7 @@ def _on_test_rows(values: npt.NDArray[np.generic]) -> npt.NDArray[np.generic]: np.where(rows.upgraded, _UPGRADED, np.where(rows.campaign_baseline, _BASELINE, _EXCLUDED)) ) - # The bin label is the same reference-derived baseline power the uplift binned on, so a row + # The bin label is the same bin power the uplift binned on, so a row # cannot sit in one bin here and another there. Outside the outer edges pd.cut gives NaN, # which is carried through as "this row belongs to no bin" rather than clipped to an edge. if "power" in self.conditions and np.isfinite(label).any(): @@ -653,8 +698,7 @@ def _on_test_rows(values: npt.NDArray[np.generic]) -> npt.NDArray[np.generic]: def _cell_membership( self, *, - rho_label: float, - ref_total: npt.NDArray[np.float64], + label: npt.NDArray[np.float64], used: npt.NDArray[np.bool_], ) -> dict[str, npt.NDArray[np.bool_]]: """Which **used** records belong to each bootstrap cell: the headline, plus each power bin. @@ -664,11 +708,11 @@ def _cell_membership( """ used_idx = np.flatnonzero(used) membership: dict[str, npt.NDArray[np.bool_]] = {_OVERALL: np.ones(len(used_idx), dtype=bool)} - if "power" not in self.conditions or not np.isfinite(rho_label): + if "power" not in self.conditions or not np.isfinite(label[used_idx]).any(): return membership assert self.rated_power_kw is not None # noqa: S101 - guaranteed by __post_init__ bins = condition_bins("power", rated_power_kw=self.rated_power_kw) - assigned = pd.cut(rho_label * ref_total[used_idx], bins=bins) + assigned = pd.cut(label[used_idx], bins=bins) for category in assigned.categories: membership[str(category)] = np.asarray(assigned == category) return membership @@ -713,16 +757,15 @@ def _conditional_frame( *, test_pw: npt.NDArray[np.float64], ref_total: npt.NDArray[np.float64], - rho_label: float, + label: npt.NDArray[np.float64], baseline: npt.NDArray[np.bool_], upgraded: npt.NDArray[np.bool_], ) -> pd.DataFrame: - """Per-power-bin uplift: ``rho_up(b) / rho_base(b) - 1``, on bins of the mean operating point. + """Per-power-bin uplift: ``rho_up(b) / rho_base(b) - 1``, on bins of ``label``. Two decisions carry this, and both are needed: - **The bin label is** ``rho_label * ref_total`` (see :func:`_rho_label`): reference-derived and - state-neutral, so neither the upgrade nor which state is called baseline can move a row + **The bin label is the bin power** (:meth:`_bin_power`): a row's own state cannot move it between bins, and it is on the test turbine's own kW scale. **The denominator is the per-bin** ``rho_base(b)``, not the global one: the test-to-reference @@ -734,7 +777,6 @@ def _conditional_frame( """ assert self.rated_power_kw is not None # noqa: S101 - guaranteed by __post_init__ bins = condition_bins("power", rated_power_kw=self.rated_power_kw) - label = rho_label * ref_total counterfactual = _per_bin_counterfactual( label=label, test_pw=test_pw, ref_total=ref_total, baseline=baseline, bins=bins ) @@ -772,6 +814,50 @@ def _toggle_period(self, timebase: pd.Timedelta) -> pd.Timedelta: raise ValueError(msg) return block + def _toggle_datum(self, mi: MethodInput, index: pd.DatetimeIndex) -> pd.Timestamp: + """Return ``toggle_datum`` after checking it is comparable with ``index`` and fits a ``ToggleSchedule``.""" + datum = pd.Timestamp(self.toggle_datum) + if (datum.tz is None) != (index.tz is None): + msg = f"{self.name}: toggle_datum {datum} and the data index must both be tz-aware or both naive" + raise ValueError(msg) + timing = mi.upgrade_timing + if isinstance(timing, ToggleSchedule) and len(index): + first = timing.start if timing.start is not None else index.min() + if (pd.Timestamp(first) - datum) % self.toggle_period != pd.Timedelta(0): + msg = ( + f"{self.name}: the ToggleSchedule tiles its cycles from {first}, which is not on the " + f"toggle_datum {datum} grid of {self.toggle_period}" + ) + raise ValueError(msg) + return datum + + def _bin_power( + self, + *, + mode: str, + test_pw: npt.NDArray[np.float64], + ref_total: npt.NDArray[np.float64], + baseline: npt.NDArray[np.bool_], + upgraded: npt.NDArray[np.bool_], + ids: npt.NDArray[np.int64] | None, + rho_base: float, + rho_up: float, + ) -> npt.NDArray[np.float64]: + """Per row, the power the bins and ``power_band`` read (``bin_power``); NaN where undefined.""" + if mode == _REFERENCE_LEG: + scale = rho_base if self.bin_power == "baseline" else 0.5 * (rho_base + rho_up) + return scale * ref_total + if ids is None: + return np.full(len(test_pw), np.nan) + + def _cycle_mean(mask: npt.NDArray[np.bool_]) -> npt.NDArray[np.float64]: + n = pd.Series(mask.astype(float)).groupby(ids).sum() + return _per_row(ids, _block_sum(ids, test_pw, mask) / n[n > 0]) + + if self.bin_power == "baseline": + return _cycle_mean(baseline) + return 0.5 * (_cycle_mean(baseline) + _cycle_mean(upgraded)) + def _reference_total( self, *, @@ -970,10 +1056,11 @@ def _write_outputs( "n_refs": len(est.refs), "n_refs_available": len(est.refs) if n_refs_available is None else n_refs_available, "refs_used": ";".join(est.refs), - "ratio_baseline": est.rho_base, - "ratio_upgraded": est.rho_up, "uplift_frc": est.uplift, "uplift_sigma_frc": est.sigma_overall, + **{f"ratio_baseline_{mode}": component.rho_base for mode, component in components.items()}, + **{f"ratio_upgraded_{mode}": component.rho_up for mode, component in components.items()}, + f"weight_{_REFERENCE_LEG}": _overall_weight(est, components), **{f"uplift_frc_{mode}": component.uplift for mode, component in components.items()}, **{f"uplift_sigma_frc_{mode}": component.sigma_overall for mode, component in components.items()}, **{f"decision_sigma_frc_{mode}": c.decision_sigma for mode, c in components.items()}, @@ -998,14 +1085,16 @@ def _write_outputs( ).to_csv(run_dir / f"{run_name}_selection_{ts}.csv", index=False) if self.save_plots: + # Scatter and ratio plots of one leg's own rows against its own reference. + shown = components.get(_REFERENCE_LEG, components[_BLOCK_LEG]) _save_plots( run_dir / "plots", wide=wide, mi=mi, test=mi.test_wtg, - used=est.used, - ref_total=est.ref_total, - reference_mode=est.mode, + used=shown.used, + ref_total=shown.ref_total, + reference_mode=shown.mode, timebase=timebase, active_power_col=self.columns.active_power, ) @@ -1065,6 +1154,8 @@ def _write_shared_diagnostics( "availability_col": self.columns.availability, "pairing_max_gap": None if self.pairing_max_gap is None else str(self.pairing_max_gap), "toggle_period": str(self.toggle_period), + "toggle_datum": str(self.toggle_datum), + "bin_power": self.bin_power, "power_band": None if self.power_band is None else list(self.power_band), "exclude_block_col": self.columns.exclude_block, } @@ -1259,13 +1350,15 @@ def _rho(test_pw: npt.NDArray[np.float64], ref_total: npt.NDArray[np.float64], m return float(test_pw[mask].sum() / denom) -def _rho_label(rho_base: float, rho_up: float) -> float: - """Return the test-to-reference ratio used to *label* bins: the mean of the two states. - - State-neutral by construction, so relabelling which state is the baseline cannot move a row - between bins. Still a campaign-level scalar, so the upgrade cannot move a row either. - """ - return 0.5 * (rho_base + rho_up) +def _overall_weight(est: _Estimate, components: dict[str, _Estimate]) -> float: + """Return the headline blend's weight on the reference leg: 1 with that leg alone, 0 without it.""" + if _REFERENCE_LEG not in components: + return 0.0 + if _BLOCK_LEG not in components: + return 1.0 + d = est.diagnostics + row = d[(d["component"] == _COMBINED) & (d["condition_bin"] == _OVERALL)] + return float(row[f"weight_{_REFERENCE_LEG}"].iloc[0]) if len(row) else float("nan") def _segment_stats( diff --git a/benchmarking/campaigns/methods.py b/benchmarking/campaigns/methods.py index e721b3ba..bf88442f 100644 --- a/benchmarking/campaigns/methods.py +++ b/benchmarking/campaigns/methods.py @@ -57,6 +57,8 @@ def carried_forward_methods( ToggleSpecialistMethod( columns=HOT_COLUMNS, toggle_period=spec.upgrade_timing.period, + # a declared campaign's schedule tiles its cycles from its start + toggle_datum=spec.upgrade_timing.start, out_dir=out_dir / "toggle_specialist", save_plots=True, conditions=("power",), diff --git a/benchmarking/harness/replicates.py b/benchmarking/harness/replicates.py index 3b41570d..b3ece1e8 100644 --- a/benchmarking/harness/replicates.py +++ b/benchmarking/harness/replicates.py @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING, Literal import numpy as np +import pandas as pd from benchmarking.harness.campaign import resolve_campaign_grid from benchmarking.harness.northing import north_scada @@ -25,7 +26,6 @@ from collections.abc import Iterator import numpy.typing as npt - import pandas as pd from benchmarking.harness.campaign import CampaignUnit from benchmarking.synthetic import ColumnSchema, SyntheticDataset, UpliftResult @@ -65,6 +65,8 @@ class StudyConfig: :param campaign_months: the campaign-length sweep grid, in months (exclusive with ``campaign_weeks``) :param campaign_weeks: the campaign-length sweep grid, in weeks (exclusive with ``campaign_months``) :param toggle_period: toggle on/off cycle length (toggle mode only) + :param toggle_datum: a boundary of the toggle-cycle grid (toggle mode only); treatment starts are + drawn only from cycle boundaries, as a clock-aligned controller toggles :param seed: RNG seed for the draws """ @@ -76,11 +78,15 @@ class StudyConfig: campaign_months: list[int] | None = None campaign_weeks: list[int] | None = None toggle_period: pd.Timedelta | None = None + toggle_datum: pd.Timestamp | None = None seed: int = 0 def __post_init__(self) -> None: - """Validate that exactly one campaign-length grid is set.""" + """Validate that exactly one campaign-length grid is set, and a toggle study's cycle grid.""" resolve_campaign_grid(campaign_months=self.campaign_months, campaign_weeks=self.campaign_weeks) + if self.mode == "toggle" and (self.toggle_period is None or self.toggle_datum is None): + msg = "toggle_period and toggle_datum are required for mode='toggle'" + raise ValueError(msg) @property def campaign_lengths(self) -> list[int]: @@ -157,7 +163,11 @@ def iter_replicates( :param rated_power_kw: turbine rating, passed to the generator and to the northing step """ subset = base_scada[base_scada[columns.turbine].isin(study.turbine_subset)] - candidates = _candidate_starts(subset.index, study.treatment_start_range) + candidates = _candidate_starts( + subset.index, + study.treatment_start_range, + cycle_grid=(study.toggle_datum, study.toggle_period) if study.mode == "toggle" else None, + ) rng = np.random.default_rng(study.seed) turbines = np.asarray(study.turbine_subset) @@ -230,22 +240,29 @@ def build_replicates( def _candidate_starts( - index: pd.DatetimeIndex, treatment_start_range: tuple[pd.Timestamp, pd.Timestamp] + index: pd.DatetimeIndex, + treatment_start_range: tuple[pd.Timestamp, pd.Timestamp], + *, + cycle_grid: tuple[pd.Timestamp | None, pd.Timedelta | None] | None = None, ) -> npt.NDArray[np.datetime64]: - """Return unique on-grid timestamps within the draw range (so draws land on real records).""" + """Return unique on-grid timestamps within the draw range (so draws land on real records). + + With ``cycle_grid`` ``(datum, period)``, only the records on a toggle-cycle boundary. + """ lo, hi = treatment_start_range unique = index.unique() candidates = unique[(unique >= lo) & (unique <= hi)] + if cycle_grid is not None: + datum, period = cycle_grid + candidates = candidates[(candidates - datum) % period == pd.Timedelta(0)] if len(candidates) == 0: - msg = f"no records in treatment_start_range {treatment_start_range}" + msg = f"no records in treatment_start_range {treatment_start_range} (on the toggle-cycle grid, if any)" raise ValueError(msg) return candidates.sort_values().to_numpy() def _upgrade_timing(study: StudyConfig, treatment_start: pd.Timestamp) -> pd.Timestamp | ToggleSchedule: if study.mode == "toggle": - if study.toggle_period is None: - msg = "toggle_period is required for mode='toggle'" - raise ValueError(msg) + assert study.toggle_period is not None # noqa: S101 - guaranteed by StudyConfig.__post_init__ return ToggleSchedule(period=study.toggle_period, start=treatment_start) return treatment_start diff --git a/benchmarking/synthetic/schema.py b/benchmarking/synthetic/schema.py index f93ff79c..b4c0ad26 100644 --- a/benchmarking/synthetic/schema.py +++ b/benchmarking/synthetic/schema.py @@ -50,9 +50,11 @@ class ColumnSchema: :param ambient_temp: ambient temperature :param exclude_row: names a caller-supplied **boolean** column marking rows to drop from row selection (e.g. special operating modes the treatment cannot affect). Optional and off by - default; when set, a method that honours it excludes the *test* turbine's flagged rows - alongside the downtime filter. Must be all-``False`` where unknown (never NaN) so an expanded - time index never turns a gap into an exclusion; a method honouring the role raises on NaN. + default; when set, a method that honours it excludes the flagged timestamps alongside the + downtime filter. Which turbines' flags count is the method's: the toggle specialist counts + the test turbine's and its references' (a flagged reference row counts the way a reference + outage does). Must be all-``False`` where unknown (never NaN) so an expanded time index + never turns a gap into an exclusion; a method honouring the role raises on NaN. :param exclude_block: names a caller-supplied **boolean** column like ``exclude_row``, but a flagged row removes the whole fixed-length block (one toggle cycle) around it, for every turbine, rather than its own timestamp alone. For a selection that must stay symmetric diff --git a/tests/benchmarking/baselines/test_toggle_end_to_end.py b/tests/benchmarking/baselines/test_toggle_end_to_end.py index 7fbe0043..b87f14b8 100644 --- a/tests/benchmarking/baselines/test_toggle_end_to_end.py +++ b/tests/benchmarking/baselines/test_toggle_end_to_end.py @@ -36,6 +36,7 @@ def test_naive_recovers_toggle_uplift(tmp_path) -> None: # noqa: ANN001 min_pre_months=24, campaign_months=[6], toggle_period=pd.Timedelta(minutes=40), + toggle_datum=pd.Timestamp("1970-01-01", tz="UTC"), n_replicates=1, seed=0, ) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 26e6e82c..fd3ba542 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -45,6 +45,8 @@ _FULLY_AVAILABLE_SECS = 3600.0 # One toggle cycle of the 20-minute ``ToggleSchedule`` most tests use (``toggle_period`` is required). _CYCLE = pd.Timedelta(minutes=20) +# Every fixture's index starts here, on the cycle grid of every period the tests use. +_DATUM = pd.Timestamp("2020-01-01", tz="UTC") # The method reads active_power + availability from the schema; the other required roles are # unused by the ratio, so name them with placeholders. _COLUMNS = ColumnSchema( @@ -121,7 +123,7 @@ def test_toggle_specialist_shares_no_wind_up_code() -> None: class TestRecovery: def test_toggle_recovers_known_uplift(self) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert isinstance(out, MethodOutput) @@ -174,7 +176,7 @@ def _varying_rho_case( def _per_bin(scada: pd.DataFrame, schedule: ToggleSchedule) -> pd.DataFrame: """Run the method with power conditioning on and return its populated per-bin rows.""" out = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None frame = out.p50_by_condition @@ -249,7 +251,7 @@ class TestConditionsConfiguration: def test_default_reports_no_conditions(self) -> None: # back-compat: existing callers get exactly today's behaviour and need no rating scada, schedule, _ = _toggle_case(uplift=0.03) - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_by_condition is None @@ -258,16 +260,16 @@ def test_power_conditioning_does_not_move_the_headline(self) -> None: # the per-bin decomposition is additional reporting, never a change to the estimate scada, schedule, _ = _toggle_case(uplift=0.03) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate(mi) + plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate(mi) conditioned = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(mi) assert conditioned.p50_overall == plain.p50_overall def test_frame_is_labelled_with_the_power_condition(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert set(out.p50_by_condition["condition"]) == {"power"} @@ -276,17 +278,27 @@ def test_frame_is_labelled_with_the_power_condition(self) -> None: def test_ws_condition_raises_citing_the_method_limit(self) -> None: with pytest.raises(ValueError, match="does not support"): - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, conditions=("ws",), rated_power_kw=_RATED_KW) + ToggleSpecialistMethod( + columns=_COLUMNS, + toggle_period=_CYCLE, + toggle_datum=_DATUM, + conditions=("ws",), + rated_power_kw=_RATED_KW, + ) def test_unknown_condition_raises(self) -> None: with pytest.raises(ValueError, match="unknown condition"): ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("bogus",), rated_power_kw=_RATED_KW + columns=_COLUMNS, + toggle_period=_CYCLE, + toggle_datum=_DATUM, + conditions=("bogus",), + rated_power_kw=_RATED_KW, ) def test_power_without_a_rating_raises(self) -> None: with pytest.raises(ValueError, match="rated_power_kw"): - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",)) + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",)) class TestPerBinSparseData: @@ -296,7 +308,7 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: # above the data's power range guarantees empty upper bins rather than hoping for them. scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=4000.0 + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=4000.0 ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None empty = out.p50_by_condition[out.p50_by_condition["n_records"] == 0] @@ -306,7 +318,7 @@ def test_bins_with_no_data_are_nan_not_imputed(self) -> None: def test_every_bin_is_represented(self) -> None: scada, schedule = _varying_rho_case(uplift=0.05) out = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert len(out.p50_by_condition) == len(condition_bins("power", rated_power_kw=_RATED_KW)) - 1 @@ -316,13 +328,18 @@ class TestPerBinDiagnostics: def test_per_bin_csv_is_written(self, tmp_path: Path) -> None: scada, schedule = _varying_rho_case(uplift=0.05) ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path + columns=_COLUMNS, + toggle_period=_CYCLE, + toggle_datum=_DATUM, + conditions=("power",), + rated_power_kw=_RATED_KW, + out_dir=tmp_path, ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert list(tmp_path.rglob("*_by_power_bin_*.csv")) def test_no_per_bin_csv_when_conditioning_is_off(self, tmp_path: Path) -> None: scada, schedule, _ = _toggle_case(uplift=0.03) - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert not list(tmp_path.rglob("*_by_power_bin_*.csv")) @@ -332,6 +349,7 @@ def test_per_bin_plot_written_with_save_plots(self, tmp_path: Path) -> None: ToggleSpecialistMethod( columns=_COLUMNS, toggle_period=_CYCLE, + toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW, out_dir=tmp_path, @@ -349,7 +367,7 @@ def test_prepost_timestamp_raises(self) -> None: treated = np.asarray(idx >= upgrade) scada = _recovery_scada(idx, treated=treated, uplift=0.05) with pytest.raises(ValueError, match="toggle"): - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=upgrade, turbine_col=_TURBINE_COL) ) @@ -366,13 +384,15 @@ def test_blank_availability_role_raises(self, blank: str) -> None: # A schema that leaves the availability role blank (empty or whitespace) would silently skip # downtime filtering, so construction must reject it. with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=replace(_COLUMNS, availability=blank), toggle_period=_CYCLE) + ToggleSpecialistMethod( + columns=replace(_COLUMNS, availability=blank), toggle_period=_CYCLE, toggle_datum=_DATUM + ) def test_missing_availability_column_raises(self) -> None: scada, schedule, _ = _toggle_case() scada = scada.drop(columns=[_AVAIL_COL]) with pytest.raises(ValueError, match="availability"): - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -385,7 +405,7 @@ def test_down_reference_timestamps_are_excluded(self) -> None: down = (corrupted[_TURBINE_COL] == "R1") & corrupted.index.isin([idx[3], idx[4]]) corrupted.loc[down, _POWER_COL] = 1e6 corrupted.loc[down, _AVAIL_COL] = 0.0 - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(0.05) @@ -395,7 +415,7 @@ class TestCompleteCase: def test_drops_timestamp_when_a_reference_is_nan(self) -> None: scada, schedule, _ = _toggle_case() idx = scada.index.unique().sort_values() - clean = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + clean = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -404,7 +424,7 @@ def test_drops_timestamp_when_a_reference_is_nan(self) -> None: corrupted = scada.copy() mask = (corrupted[_TURBINE_COL] == "R1") & (corrupted.index == idx[3]) corrupted.loc[mask, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(clean.p50_overall) @@ -415,12 +435,12 @@ def test_nan_at_unused_timestamp_does_not_change_estimate(self) -> None: # make idx[2] already unused (test NaN), then add a second NaN at the same timestamp base = scada.copy() base.loc[(base[_TURBINE_COL] == "T1") & (base.index == idx[2]), _POWER_COL] = np.nan - before = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + before = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=base, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) after_df = base.copy() after_df.loc[(after_df[_TURBINE_COL] == "R1") & (after_df.index == idx[2]), _POWER_COL] = np.nan - after = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + after = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=after_df, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert after.p50_overall == pytest.approx(before.p50_overall) @@ -435,7 +455,7 @@ def test_estimate_invariant_to_timebase(self) -> None: treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.04) results.append( - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=schedule.period) + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=schedule.period, toggle_datum=_DATUM) .estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) .p50_overall ) @@ -446,6 +466,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 method = ToggleSpecialistMethod( columns=_COLUMNS, toggle_period=pd.Timedelta(hours=1), + toggle_datum=_DATUM, out_dir=tmp_path, timebase=pd.Timedelta(minutes=30), ) @@ -460,7 +481,7 @@ def test_override_timebase_used_for_mwh(self, tmp_path) -> None: # noqa: ANN001 class TestActivePowerOnly: def test_extra_columns_ignored(self) -> None: scada, schedule, _ = _toggle_case() - plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + plain = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -468,7 +489,7 @@ def test_extra_columns_ignored(self) -> None: rng = np.random.default_rng(0) with_extra["ws"] = rng.normal(size=len(with_extra)) with_extra["rpm"] = rng.normal(size=len(with_extra)) - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=with_extra, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.p50_overall == pytest.approx(plain.p50_overall) @@ -480,7 +501,7 @@ def test_no_reference_is_the_block_leg_alone(self, caplog: pytest.LogCaptureFixt schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=idx[0]) scada = _scada({"T1": np.ones(len(idx))}, idx) with caplog.at_level("WARNING"): - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert _legs(out) == {"block_mean"} @@ -493,7 +514,7 @@ def test_no_used_baseline_returns_nan(self) -> None: off_ts = idx[~treated] is_baseline_test = (scada[_TURBINE_COL] == "T1") & scada.index.isin(off_ts) scada.loc[is_baseline_test, _POWER_COL] = np.nan - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert np.isnan(out.p50_overall) @@ -546,7 +567,9 @@ def _read_only_csv(folder: Path, kind: str) -> pd.DataFrame: class TestDiagnostics: def _run(self, tmp_path, *, save_plots: bool = False): # noqa: ANN001, ANN202 scada, schedule, _ = _toggle_case(uplift=0.06) - method = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=save_plots) + method = ToggleSpecialistMethod( + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path, save_plots=save_plots + ) out = method.estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -556,15 +579,28 @@ def test_writes_both_csvs(self, tmp_path) -> None: # noqa: ANN001 self._run(tmp_path) stats = _read_only_csv(tmp_path, "data_stats") results = _read_only_csv(tmp_path, "results") - assert sorted(stats["segment"]) == ["all", "baseline", "upgraded"] + for _, leg in stats.groupby("component"): + assert sorted(leg["segment"]) == ["all", "baseline", "upgraded"] + assert set(stats["component"]) == {"sum", "block_mean"} assert len(results) == 1 def test_uplift_rederivable_from_stats(self, tmp_path) -> None: # noqa: ANN001 + """Each leg's uplift from its own stats rows; the headline is their blend at the reported weight.""" out, _ = self._run(tmp_path) - stats = _read_only_csv(tmp_path, "data_stats").set_index("segment") - rho_base = stats.loc["baseline", "used_test_mwh"] / stats.loc["baseline", "used_ref_total_mwh"] - rho_up = stats.loc["upgraded", "used_test_mwh"] / stats.loc["upgraded", "used_ref_total_mwh"] - assert (rho_up / rho_base - 1.0) == pytest.approx(out.p50_overall) + stats = _read_only_csv(tmp_path, "data_stats").set_index(["component", "segment"]) + results = _read_only_csv(tmp_path, "results").iloc[0] + leg_uplift = {} + for leg in ("sum", "block_mean"): + rho_base = ( + stats.loc[(leg, "baseline"), "used_test_mwh"] / stats.loc[(leg, "baseline"), "used_ref_total_mwh"] + ) + rho_up = stats.loc[(leg, "upgraded"), "used_test_mwh"] / stats.loc[(leg, "upgraded"), "used_ref_total_mwh"] + leg_uplift[leg] = rho_up / rho_base - 1.0 + assert leg_uplift[leg] == pytest.approx( + results[f"ratio_upgraded_{leg}"] / results[f"ratio_baseline_{leg}"] - 1.0 + ) + weight = results["weight_sum"] + assert weight * leg_uplift["sum"] + (1.0 - weight) * leg_uplift["block_mean"] == pytest.approx(out.p50_overall) def test_results_csv_matches_estimate(self, tmp_path) -> None: # noqa: ANN001 out, _ = self._run(tmp_path) @@ -576,7 +612,7 @@ def test_results_csv_matches_estimate(self, tmp_path) -> None: # noqa: ANN001 def test_stats_coverage_full_for_complete_data(self, tmp_path) -> None: # noqa: ANN001 self._run(tmp_path) - stats = _read_only_csv(tmp_path, "data_stats").set_index("segment") + stats = _read_only_csv(tmp_path, "data_stats").query("component == 'sum'").set_index("segment") assert stats.loc["all", "rows_data_coverage"] == pytest.approx(1.0) assert stats.loc["all", "used_data_coverage"] == pytest.approx(1.0) assert stats.loc["all", "n_used_timestamps"] == 40 @@ -605,12 +641,13 @@ def test_precampaign_dropped_from_baseline(self, tmp_path) -> None: # noqa: ANN schedule = ToggleSchedule(period=pd.Timedelta(minutes=20), start=start) treated = np.asarray(treated_mask(idx, schedule)) scada = _recovery_scada(idx, treated=treated, uplift=0.05) - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - baseline = _read_only_csv(tmp_path, "data_stats").set_index("segment").loc["baseline"] + stats = _read_only_csv(tmp_path, "data_stats") # the baseline class starts at/after the campaign start; the pre-campaign rows are excluded. - assert pd.Timestamp(baseline["first_timestamp"]) >= start + for first in stats.loc[stats["segment"] == "baseline", "first_timestamp"]: + assert pd.Timestamp(first) >= start # --- uncertainty (non-optional, computed after the uplift) -------------------------------------- @@ -643,6 +680,7 @@ def _long_diagnostics(out: MethodOutput) -> pd.DataFrame: def _estimate(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: kwargs.setdefault("toggle_period", _CYCLE) + kwargs.setdefault("toggle_datum", _DATUM) return ToggleSpecialistMethod(columns=_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -733,7 +771,7 @@ def test_a_nan_uplift_reports_a_nan_sigma(self) -> None: idx = _index(40) # every row treated -> no off rows -> rho_base is NaN scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput( scada_df=scada, test_wtg="T1", @@ -750,7 +788,7 @@ def test_the_bootstrap_is_not_run_when_the_uplift_is_nan(self) -> None: idx = _index(40) scada = _recovery_scada(idx, treated=np.ones(len(idx), dtype=bool)) out = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ).estimate( MethodInput( scada_df=scada, @@ -839,7 +877,7 @@ def test_one_row_per_test_turbine_record(self) -> None: def test_used_reproduces_the_methods_own_used_mask(self) -> None: scada, schedule = _noisy_toggle_case() method = ToggleSpecialistMethod( - columns=_COLUMNS, toggle_period=_CYCLE, conditions=("power",), rated_power_kw=_RATED_KW + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW ) mi = MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) out = method.estimate(mi) @@ -936,20 +974,20 @@ def _swap_states(toggle_df: pd.DataFrame) -> pd.DataFrame: class TestReversalSymmetry: - """Which of two toggle states is *called* the baseline is a naming choice, not a measurement. + """With ``bin_power="both_states"``, which of two toggle states is *called* the baseline is a naming choice. - The bin label must therefore not depend on it. It used to: the label was the baseline state's - predicted power, so renaming the states shifted every label by the uplift and migrated rows - across bin edges -- moving per-bin estimates by an appreciable fraction of their own sigma for - no physical reason. + The bin label must then not depend on it. Under ``"baseline"`` it does by design (the label is + the baseline state's power): renaming the states shifts every label by the uplift and migrates + rows across bin edges, which is why a campaign with no true baseline state uses ``both_states``. """ def _pair(self) -> tuple[MethodOutput, MethodOutput]: scada, schedule = _varying_rho_case(n=600, uplift=0.05) index = pd.DatetimeIndex(pd.unique(scada.index)).sort_values() toggle_df = build_toggle_df(index, schedule) - forward = _estimate(scada, toggle_df, conditions=("power",), rated_power_kw=_RATED_KW) - reversed_ = _estimate(scada, _swap_states(toggle_df), conditions=("power",), rated_power_kw=_RATED_KW) + kwargs = {"conditions": ("power",), "rated_power_kw": _RATED_KW, "bin_power": "both_states"} + forward = _estimate(scada, toggle_df, **kwargs) + reversed_ = _estimate(scada, _swap_states(toggle_df), **kwargs) return forward, reversed_ def test_every_row_keeps_its_power_bin_when_the_states_are_swapped(self) -> None: @@ -1002,7 +1040,7 @@ class TestExcludeRow: def test_flagged_test_rows_are_not_used(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1018,10 +1056,10 @@ def test_excluding_corrupted_rows_restores_the_known_uplift(self) -> None: spoiled = corrupted[_EXCLUDE_COL].to_numpy(dtype=bool) & (corrupted[_TURBINE_COL] == "T1").to_numpy() corrupted.loc[spoiled, _POWER_COL] *= 0.5 - naive = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + naive = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) - filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( + filtered = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=corrupted, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert filtered.p50_overall == pytest.approx(0.05, abs=1e-9) @@ -1038,9 +1076,9 @@ def test_flagged_reference_rows_leave_the_reference_leg_like_an_outage(self, tmp flagged[_EXCLUDE_COL] = ( flagged[_EXCLUDE_COL].to_numpy() | _flag(scada, turbine="R2", every=3)[_EXCLUDE_COL].to_numpy() ) - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path).estimate( - MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + out = ToggleSpecialistMethod( + columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path + ).estimate(MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.labeled_rows is not None is_ref = (flagged[_TURBINE_COL] != "T1").to_numpy() flagged_ts = flagged.index[is_ref & flagged[_EXCLUDE_COL].to_numpy(dtype=bool)].unique() @@ -1056,14 +1094,14 @@ def test_nan_in_the_exclude_column_is_rejected(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=7, value=np.nan) with pytest.raises(ValueError, match=_EXCLUDE_COL): - ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( + ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) def test_unset_role_excludes_nothing(self) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) - out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1073,7 +1111,7 @@ def test_unset_role_excludes_nothing(self) -> None: def test_absent_column_excludes_nothing(self) -> None: """The role names a column the frame does not carry: skip, do not raise.""" scada, schedule = _noisy_toggle_case() - out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE).estimate( + out = ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) assert out.labeled_rows is not None @@ -1089,7 +1127,7 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: scada, schedule = _noisy_toggle_case() flagged = _flag(scada, turbine="T1", every=5) ToggleSpecialistMethod( - columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=True + columns=_EXCLUDE_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path, save_plots=True ).estimate(MethodInput(scada_df=flagged, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) names = {p.name for p in (run_dir / "plots").rglob("*.png")} @@ -1098,9 +1136,9 @@ def test_exclusions_reach_the_shared_diagnostics(self, tmp_path: Path) -> None: def test_no_exclusion_plots_when_nothing_is_excluded(self, tmp_path: Path) -> None: scada, schedule = _noisy_toggle_case() - ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, out_dir=tmp_path, save_plots=True).estimate( - MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) - ) + ToggleSpecialistMethod( + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, out_dir=tmp_path, save_plots=True + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) run_dir = next(p for p in Path(tmp_path).iterdir() if p.is_dir()) names = {p.name for p in (run_dir / "plots").rglob("*.png")} assert "ops_curves_excluded.png" not in names @@ -1134,7 +1172,7 @@ def _fixture() -> tuple[pd.DataFrame, ToggleSchedule]: @staticmethod def _estimate(scada: pd.DataFrame, **kwargs: object) -> float: - method = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE) + method = ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM) return method.estimate(MethodInput(scada_df=scada, **kwargs)).p50_overall def test_only_offered_references_are_used(self) -> None: @@ -1259,6 +1297,7 @@ def _block_excluded_case() -> tuple[pd.DataFrame, ToggleSchedule, pd.DatetimeInd @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: kwargs.setdefault("toggle_period", _CYCLE) + kwargs.setdefault("toggle_datum", _DATUM) return ToggleSpecialistMethod(columns=_EXCLUDE_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) @@ -1362,8 +1401,39 @@ def _legs(out: MethodOutput) -> set[str]: return set(_long_diagnostics(out)["component"]) +class TestBinPower: + """``bin_power`` picks the per-row power the bins and ``power_band`` read.""" + + @staticmethod + def _bins(uplift: float, bin_power: str) -> pd.Series: + out = _estimate( + *_varying_rho_case(n=600, uplift=uplift), + conditions=("power",), + rated_power_kw=_RATED_KW, + bin_power=bin_power, + ) + assert out.labeled_rows is not None + return out.labeled_rows["power_bin"].astype(str) + + def test_baseline_bins_do_not_move_with_the_upgraded_rows(self) -> None: + assert (self._bins(0.0, "baseline") == self._bins(0.10, "baseline")).all() + + def test_both_states_bins_move_with_the_upgraded_rows(self) -> None: + """The control for the test above: the same change does move the both-states label.""" + assert (self._bins(0.0, "both_states") != self._bins(0.10, "both_states")).any() + + def test_an_unknown_bin_power_is_refused(self) -> None: + with pytest.raises(ValueError, match="bin_power"): + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, bin_power="mean") # type: ignore[arg-type] + + class TestPowerBand: - """``power_band`` keeps the used rows whose reference-derived baseline power lies in ``(lo, hi]``.""" + """``power_band`` keeps the used rows whose bin power lies in ``(lo, hi]``.""" + + @pytest.mark.parametrize("band", [(800.0, 400.0), (400.0, 400.0), (0.0, float("inf"))]) + def test_a_band_that_is_not_a_finite_interval_is_refused(self, band: tuple[float, float]) -> None: + with pytest.raises(ValueError, match="power_band"): + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, power_band=band) _BAND = (360.0, 840.0) # two whole 240 kW bins at 1200 kW rated @@ -1416,6 +1486,31 @@ def test_a_block_off_the_timebase_grid_is_refused(self) -> None: _estimate(scada, schedule, toggle_period=pd.Timedelta(minutes=25)) +class TestToggleDatum: + """``toggle_datum`` is the caller's statement of the cycle grid: required, never taken from the data.""" + + def test_it_is_required(self) -> None: + with pytest.raises(TypeError, match="toggle_datum"): + ToggleSpecialistMethod(columns=_COLUMNS, toggle_period=_CYCLE) # type: ignore[call-arg] + + def test_cycles_stay_on_the_datum_grid_when_the_first_record_is_missing(self) -> None: + """Tiling from the first record present would put every cycle across two power levels.""" + scada, schedule = _block_case(uplift=0.04) + scada = scada[scada.index != scada.index.min()] + out = _estimate(scada, schedule, toggle_period=_BLOCK) + assert out.p50_overall == pytest.approx(0.04, abs=1e-3) + + def test_a_schedule_off_the_datum_grid_is_refused(self) -> None: + scada, schedule = _block_case(n_blocks=10) + with pytest.raises(ValueError, match="toggle_datum"): + _estimate(scada, schedule, toggle_period=_BLOCK, toggle_datum=_DATUM + pd.Timedelta(minutes=10)) + + def test_a_naive_datum_on_tz_aware_data_is_refused(self) -> None: + scada, schedule = _block_case(n_blocks=10) + with pytest.raises(ValueError, match="tz-aware"): + _estimate(scada, schedule, toggle_period=_BLOCK, toggle_datum=_DATUM.tz_localize(None)) + + class TestBlockLeg: """The block leg: the test turbine against its own block's mean power, needing no references.""" @@ -1535,6 +1630,7 @@ class TestExcludeBlock: @staticmethod def _run(scada: pd.DataFrame, schedule: ToggleSchedule, **kwargs: object) -> MethodOutput: + kwargs.setdefault("toggle_datum", _DATUM) return ToggleSpecialistMethod(columns=_BLOCK_COLUMNS, **kwargs).estimate( MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL) ) diff --git a/tests/benchmarking/harness/test_replicates.py b/tests/benchmarking/harness/test_replicates.py index 6aa1ad72..dad2d32b 100644 --- a/tests/benchmarking/harness/test_replicates.py +++ b/tests/benchmarking/harness/test_replicates.py @@ -42,6 +42,7 @@ def _study(mode: str = "prepost", n_replicates: int = 5, seed: int = 0) -> Study min_pre_months=12, campaign_months=[3, 6], toggle_period=pd.Timedelta(days=14), + toggle_datum=pd.Timestamp("2017-01-01", tz="UTC"), n_replicates=n_replicates, seed=seed, ) @@ -55,6 +56,7 @@ def _weeks_study(mode: str = "toggle", n_replicates: int = 5, seed: int = 0) -> min_pre_months=12, campaign_weeks=[1, 2, 4, 8], toggle_period=pd.Timedelta(days=14), + toggle_datum=pd.Timestamp("2017-01-01", tz="UTC"), n_replicates=n_replicates, seed=seed, ) @@ -106,6 +108,32 @@ def test_max_activity_months_raises_for_a_weeks_study(self) -> None: _ = _weeks_study().max_activity_months +class TestToggleCycleGrid: + """A toggle study draws treatment starts only on its cycle grid, so one datum serves every replicate.""" + + def test_a_toggle_study_needs_a_datum(self) -> None: + with pytest.raises(ValueError, match="toggle_datum"): + StudyConfig( + mode="toggle", + turbine_subset=["T1"], + treatment_start_range=(pd.Timestamp("2017-01-01", tz="UTC"), pd.Timestamp("2017-12-31", tz="UTC")), + min_pre_months=12, + campaign_weeks=[2], + toggle_period=pd.Timedelta(days=14), + n_replicates=1, + ) + + def test_every_drawn_start_is_a_cycle_boundary(self) -> None: + study = _weeks_study(n_replicates=8) + reps = build_replicates(_base_scada(), profile=PROFILE, study=study) + assert study.toggle_period is not None + assert study.toggle_datum is not None + for rep in reps: + start = pd.Timestamp(rep.treatment_start) + start = start if start.tz is not None else start.tz_localize("UTC") + assert (start - study.toggle_datum) % study.toggle_period == pd.Timedelta(0) + + def test_build_replicates_returns_n_replicate_records() -> None: reps = build_replicates(_base_scada(), profile=PROFILE, study=_study(n_replicates=5)) assert len(reps) == 5 diff --git a/tests/benchmarking/harness/test_scoring.py b/tests/benchmarking/harness/test_scoring.py index 6e23aa24..12b978bb 100644 --- a/tests/benchmarking/harness/test_scoring.py +++ b/tests/benchmarking/harness/test_scoring.py @@ -46,6 +46,7 @@ def _study(mode: str = "prepost", n_replicates: int = 4, seed: int = 0) -> Study min_pre_months=12, campaign_months=[3, 6], toggle_period=pd.Timedelta(days=14), + toggle_datum=pd.Timestamp("2017-01-01", tz="UTC"), n_replicates=n_replicates, seed=seed, ) From 8bcbc2826a4dce095ed2a3fc0b8fa5b72b6d0097 Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Wed, 7 Oct 2026 17:20:16 +0100 Subject: [PATCH 17/18] ToggleSpecialistMethod: per-bin energy sums per leg, with each bin's blend weight The blended per-bin frame took sum_actual / sum_counterfactual from one leg while p50_uplift is the blend. Each leg's frame now carries sum_actual_ / sum_counterfactual_, and the blended frame carries both legs' and the bin's weight_sum, so every bin's blend is re-derivable. Co-Authored-By: Claude Opus 5.5 --- benchmarking/baselines/toggle_specialist.py | 14 ++++---- .../baselines/test_toggle_specialist.py | 32 +++++++++++++++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index c125ad95..922d0d3e 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -529,7 +529,7 @@ def _component( label=label, baseline=used & baseline, upgraded=used & rows.upgraded, - ) + ).rename(columns={"sum_actual": f"sum_actual_{mode}", "sum_counterfactual": f"sum_counterfactual_{mode}"}) if "power" in self.conditions else None ) @@ -771,7 +771,8 @@ def _conditional_frame( **The denominator is the per-bin** ``rho_base(b)``, not the global one: the test-to-reference ratio varies with power, and a global denominator would read that structure as uplift. The price is that the per-bin numbers no longer aggregate exactly to ``p50_overall``, which is - deliberate and un-relevelled; ``sum_actual`` / ``sum_counterfactual`` expose the gap. + deliberate and un-relevelled; the leg's ``sum_actual_`` / ``sum_counterfactual_`` + expose the gap. Sparse bins report NaN with ``n_records = 0`` rather than being imputed. """ @@ -1194,7 +1195,8 @@ def _combine(a: _Estimate, b: _Estimate, *, upgraded: npt.NDArray[np.bool_], bin The blend's used rows are the union of the components' (a row contributed to at least one estimate); its bin label is ``a``'s where ``a`` used the row and ``b``'s otherwise, so the two components' bins, which share edges, are merged by name, and each bin's ``n_records`` counts - the used ``upgraded`` rows that merged label puts in it (what ``labeled_rows`` reproduces). + the used ``upgraded`` rows that merged label puts in it (what ``labeled_rows`` reproduces). Each + bin also carries both legs' energy sums and its weight on ``a``, so its blend is re-derivable. Row-selection accounting, ``rho`` and the reference-side diagnostics are ``a``'s (the reference leg); ``b``'s selection is written to the selection CSV. The diagnostics carry both legs' rows and a ``combined`` row per cell with the blend's sigma, the weight on ``a`` and the correlation it used. @@ -1234,15 +1236,15 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> ra, rb = fa.loc[cell], fb.loc[cell] blend = _cell(cell, ra["p50_uplift"], ra["sigma_uplift"], rb["p50_uplift"], rb["sigma_uplift"]) blended_cells[cell] = blend - take = ra if np.isfinite(ra["p50_uplift"]) else rb records.append( { "condition": ra["condition"], "condition_bin": ra["condition_bin"], "p50_uplift": blend.estimate, "n_records": int((upgraded_bin == cell).sum()), - "sum_actual": take["sum_actual"], - "sum_counterfactual": take["sum_counterfactual"], + **{f"{col}_{a.mode}": ra[f"{col}_{a.mode}"] for col in ("sum_actual", "sum_counterfactual")}, + **{f"{col}_{b.mode}": rb[f"{col}_{b.mode}"] for col in ("sum_actual", "sum_counterfactual")}, + f"weight_{a.mode}": blend.weight_a, "sigma_uplift": blend.sigma, } ) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index fd3ba542..53fb63d8 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -242,7 +242,7 @@ def test_uplift_confined_to_high_power_shows_only_there(self) -> None: scada = _scada({"T1": test, "R1": ref1, "R2": ref2}, idx) per_bin = _per_bin(scada, schedule) - low_bins = per_bin[per_bin["sum_counterfactual"] > 0].iloc[:2] + low_bins = per_bin[per_bin["sum_counterfactual_sum"] > 0].iloc[:2] assert low_bins["p50_uplift"].abs().max() < 0.005 assert per_bin["p50_uplift"].max() == pytest.approx(0.10, abs=0.01) @@ -273,8 +273,36 @@ def test_frame_is_labelled_with_the_power_condition(self) -> None: ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) assert out.p50_by_condition is not None assert set(out.p50_by_condition["condition"]) == {"power"} - for col in ("condition_bin", "p50_uplift", "n_records", "sum_actual", "sum_counterfactual"): + for col in ("condition_bin", "p50_uplift", "n_records", "sigma_uplift", "weight_sum"): assert col in out.p50_by_condition.columns + for leg in ("sum", "block_mean"): + assert {f"sum_actual_{leg}", f"sum_counterfactual_{leg}"} <= set(out.p50_by_condition.columns) + assert "sum_actual" not in out.p50_by_condition.columns + + def test_each_bin_is_the_blend_of_its_legs_at_its_weight(self) -> None: + """Each leg's bin uplift is its own energy ratio; the reported bin is their blend at the bin's weight.""" + scada, schedule = _varying_rho_case(uplift=0.05, noise_frac=0.05) + out = ToggleSpecialistMethod( + columns=_COLUMNS, toggle_period=_CYCLE, toggle_datum=_DATUM, conditions=("power",), rated_power_kw=_RATED_KW + ).estimate(MethodInput(scada_df=scada, test_wtg="T1", upgrade_timing=schedule, turbine_col=_TURBINE_COL)) + assert out.p50_by_condition is not None + frame = out.p50_by_condition.dropna(subset=["p50_uplift", "weight_sum"]) + assert len(frame) + legs = { + leg: frame[f"sum_actual_{leg}"] / frame[f"sum_counterfactual_{leg}"] - 1.0 for leg in ("sum", "block_mean") + } + weight = frame["weight_sum"] + # a leg with no estimate in a bin carries no weight there + blended = np.where(weight > 0, weight * legs["sum"], 0.0) + np.where( + weight < 1, (1.0 - weight) * legs["block_mean"], 0.0 + ) + np.testing.assert_allclose(frame["p50_uplift"], blended) + + def test_a_single_leg_frame_carries_that_legs_sums(self) -> None: + scada, schedule = _block_case(uplift=0.04) + out = _estimate(scada, schedule, toggle_period=_BLOCK, conditions=("power",), rated_power_kw=_RATED_KW) + assert out.p50_by_condition is not None + assert {"sum_actual_block_mean", "sum_counterfactual_block_mean"} <= set(out.p50_by_condition.columns) def test_ws_condition_raises_citing_the_method_limit(self) -> None: with pytest.raises(ValueError, match="does not support"): From 201e07cd7014b336bb1780bbb675e875ada8ecac Mon Sep 17 00:00:00 2001 From: Alex Clerc Date: Thu, 8 Oct 2026 10:45:38 +0100 Subject: [PATCH 18/18] ToggleSpecialistMethod: drop degenerate bootstrap resamples, report each leg's uplift A block-bootstrap resample whose on-side or off-side reference energy is below a tenth of the whole campaign's (a calm spell or outage drawn over and over) now counts as non-finite instead of feeding the spread: on a near-calm fortnight the reported sigma reached 10^5 %. Every row of uncertainty_diagnostics carries uplift, per leg and for the blend, overall and per bin, so a consumer can see which leg an error came from without re-running. Co-Authored-By: Claude Fable 5.1 --- benchmarking/baselines/block_bootstrap.py | 14 +++++++++++--- benchmarking/baselines/toggle_specialist.py | 12 +++++++++++- .../benchmarking/baselines/test_block_bootstrap.py | 13 +++++++++++++ .../baselines/test_toggle_specialist.py | 6 ++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/benchmarking/baselines/block_bootstrap.py b/benchmarking/baselines/block_bootstrap.py index 02f08a11..b3f15c5b 100644 --- a/benchmarking/baselines/block_bootstrap.py +++ b/benchmarking/baselines/block_bootstrap.py @@ -32,6 +32,9 @@ # Per (cell, segment): the numerator and denominator of each segment's rho. _N_QUANTITIES = 4 _MIN_RESAMPLES_FOR_SPREAD = 2 +# A resample whose reference energy on either side falls below this share of the whole campaign's is a +# degenerate ratio (a calm spell or outage drawn over and over), not a draw from the estimator's spread. +_MIN_REFERENCE_FRACTION = 0.1 # With one block covering the whole campaign, every resample is that campaign: nothing varies. _MIN_BLOCKS_FOR_SPREAD = 2 # Normal -/+1 sigma percentiles, so (p84 - p16) / 2 is a sigma for a normal. @@ -328,7 +331,7 @@ def bootstrap_ratio_uplift( # vary. Any sigma it returned would be float residue (~1e-15), not a real certainty. return BootstrapResult(n_blocks=n_blocks, cells=_fallback_only_cells(names, fallback=fallback)) - uplift = _uplift_from_totals(totals) + uplift = _uplift_from_totals(totals, reference_floor=_MIN_REFERENCE_FRACTION * prefix[n_records]) boot_weight = {name: _bootstrap_weight(min(on, off)) for name, (on, off) in counts.items()} return BootstrapResult( n_blocks=n_blocks, @@ -387,17 +390,22 @@ def _prefix_sums( return np.concatenate([np.zeros((1, len(names), _N_QUANTITIES)), np.cumsum(doubled, axis=0)], axis=0) -def _uplift_from_totals(totals: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: +def _uplift_from_totals( + totals: npt.NDArray[np.float64], *, reference_floor: npt.NDArray[np.float64] | None = None +) -> npt.NDArray[np.float64]: """Re-form ``rho_up / rho_base - 1`` per (resample, cell) from resampled sums. The degeneracy guards mirror the point estimate's, so a resample fails only where the point - estimate would have failed on the same rows. + estimate would have failed on the same rows; ``reference_floor`` (per cell, same last axis as + ``totals``) additionally fails a resample whose on or off reference sum falls below it. """ test_on, ref_on, test_off, ref_off = (totals[..., k] for k in range(_N_QUANTITIES)) nan = np.full(test_on.shape, np.nan) rho_up = np.divide(test_on, ref_on, out=nan.copy(), where=ref_on != 0) rho_base = np.divide(test_off, ref_off, out=nan.copy(), where=ref_off != 0) valid = np.isfinite(rho_base) & (rho_base != 0) & np.isfinite(rho_up) + if reference_floor is not None: + valid &= (ref_on >= reference_floor[..., 1]) & (ref_off >= reference_floor[..., 3]) return np.divide(rho_up, rho_base, out=nan.copy(), where=valid) - 1.0 diff --git a/benchmarking/baselines/toggle_specialist.py b/benchmarking/baselines/toggle_specialist.py index 922d0d3e..60a976fd 100644 --- a/benchmarking/baselines/toggle_specialist.py +++ b/benchmarking/baselines/toggle_specialist.py @@ -93,6 +93,8 @@ from benchmarking.synthetic import ToggleSchedule if TYPE_CHECKING: + from collections.abc import Mapping + import numpy.typing as npt from benchmarking.synthetic import ColumnSchema @@ -562,14 +564,19 @@ def _run_bootstrap(up: npt.NDArray[np.bool_], base: npt.NDArray[np.bool_]) -> Bo rng=np.random.default_rng(self.bootstrap_seed), ) decision_boot = _run_bootstrap(up_p, base_p) + uplift_by_cell = {_OVERALL: uplift} if per_bin is not None: per_bin["sigma_uplift"] = [_cell_sigma(boot, str(b)) for b in per_bin["condition_bin"]] + uplift_by_cell.update( + {str(b): float(u) for b, u in zip(per_bin["condition_bin"], per_bin["p50_uplift"], strict=True)} + ) diagnostics = _uncertainty_diagnostics( boot, membership=membership, upgraded=used & rows.upgraded, baseline=used & baseline, used=used, + uplift_by_cell=uplift_by_cell, ) diagnostics.insert(0, "component", mode) return _Estimate( @@ -1256,6 +1263,7 @@ def _cell(cell: str, est_a: float, sig_a: float, est_b: float, sig_b: float) -> "component": _COMBINED, "condition": _OVERALL if cell == _OVERALL else "power", "condition_bin": cell, + "uplift": blend.estimate, "sigma": blend.sigma, f"weight_{a.mode}": blend.weight_a, "correlation": blend.correlation, @@ -1308,8 +1316,9 @@ def _uncertainty_diagnostics( upgraded: npt.NDArray[np.bool_], baseline: npt.NDArray[np.bool_], used: npt.NDArray[np.bool_], + uplift_by_cell: Mapping[str, float] | None = None, ) -> pd.DataFrame: - """Per-cell account of how the uncertainty was reached, keyed by ``(condition, condition_bin)``. + """Per-cell account of the estimate and how its uncertainty was reached, keyed by ``(condition, condition_bin)``. Carried through the harness seam uninterpreted, so an uncertainty model can be developed against a saved sweep rather than by re-running one. Emitted even when the bootstrap did not run: the @@ -1327,6 +1336,7 @@ def _uncertainty_diagnostics( { "condition": _OVERALL if cell == _OVERALL else "power", "condition_bin": cell, + "uplift": uplift_by_cell.get(cell, nan) if uplift_by_cell is not None else nan, "n_upgraded_records": int((member & up_used).sum()), "n_baseline_records": int((member & base_used).sum()), "n_blocks": boot.n_blocks if boot is not None else 0, diff --git a/tests/benchmarking/baselines/test_block_bootstrap.py b/tests/benchmarking/baselines/test_block_bootstrap.py index 418a72e1..61dc8269 100644 --- a/tests/benchmarking/baselines/test_block_bootstrap.py +++ b/tests/benchmarking/baselines/test_block_bootstrap.py @@ -207,6 +207,19 @@ def test_no_records_gives_nan_cells(self) -> None: assert result.n_blocks == 0 assert np.isnan(result.cells["overall"].sigma) + def test_a_resample_of_almost_no_reference_energy_is_not_a_spread_sample(self) -> None: + """A calm spell drawn over and over divides by nearly nothing; those resamples are dropped, not believed.""" + case = _case(n=4032) + calm = np.arange(len(case["times"])) < int(0.9 * len(case["times"])) + case["ref_total"] = np.where(calm, 1e-3, case["ref_total"]) + case["test_power"] = np.where( + calm, 1e-3 * (1.0 + 0.5 * np.random.default_rng(1).standard_normal(len(calm))), case["test_power"] + ) + cell = _run(case).cells["overall"] + assert np.isfinite(cell.sigma) + assert cell.sigma < 1.0 + assert cell.frac_resamples_finite < 1.0 + def test_too_few_resamples_for_a_spread_gives_nan(self) -> None: assert np.isnan(_run(_case(), n_resamples=1).cells["overall"].sigma) diff --git a/tests/benchmarking/baselines/test_toggle_specialist.py b/tests/benchmarking/baselines/test_toggle_specialist.py index 53fb63d8..9379b395 100644 --- a/tests/benchmarking/baselines/test_toggle_specialist.py +++ b/tests/benchmarking/baselines/test_toggle_specialist.py @@ -1614,6 +1614,12 @@ def test_diagnostics_carry_both_legs_and_the_blend(self) -> None: assert 0.0 <= head["weight_sum"] <= 1.0 assert np.isfinite(head["decision_sigma_sum"]) assert np.isfinite(head["decision_sigma_block_mean"]) + assert head["uplift"] == pytest.approx(out.p50_overall) + legs = diag[(diag["component"] != "combined") & (diag["condition_bin"] == "overall")] + assert np.isfinite(legs["uplift"]).all() + w = head["weight_sum"] + by_leg = legs.set_index("component")["uplift"] + assert out.p50_overall == pytest.approx(w * by_leg["sum"] + (1 - w) * by_leg["block_mean"]) def test_the_blend_is_the_weighted_legs_and_its_sigma_comes_from_the_actual_bootstraps( self, tmp_path: Path