band_ladder

import matplotlib.pyplot as plt
 
from _setup import X_test, artifact, y_test
 
from compileml.runtime import decide
from compileml.viz import band_ladder
 
# The whole holdout, not a slice of it. Empirical monotonicity is a question
# about sample size as much as about the model: at ~200 accounts per band the
# standard error on a 10% rate is about 0.021, so adjacent bands routinely
# cross for no reason anyone should act on.
decisions = [decide(artifact, row.tolist(), explain=False) for row in X_test]
 
fig, ax = band_ladder(decisions, y_test)
fig.savefig("../../public/examples/band_ladder.png", dpi=144, bbox_inches="tight")
plt.close(fig)
 
counts, bad = {}, {}
for decision, outcome in zip(decisions, y_test):
    band = decision["band"]
    counts[band] = counts.get(band, 0) + 1
    bad[band] = bad.get(band, 0) + int(outcome)
 
labels = sorted(counts)
rates = [bad[b] / counts[b] for b in labels]
 
print(f"{'band':<7}{'n':>7}{'bad rate':>11}")
for label, rate in zip(labels, rates):
    print(f"{label:<7}{counts[label]:>7}{rate:>11.4f}")
 
inversions = [
    (labels[i], labels[i + 1]) for i in range(len(rates) - 1) if rates[i + 1] < rates[i]
]
print()
print(f"rows scored          {len(decisions)}")
print(f"empirical inversions {len(inversions)}")
 
# Monotone band edges are built to hold on the data they were fit on. Whether
# they hold out of sample is measured, not assumed — sweep_bands reports
# monotonicity_violations per candidate ladder, and band_efficiency gives a
# per-band verdict on whether a band still separates risk internally.