 codex/reliability-diagram-bin-merging.py

"""Synthetic calibration example; Python 3.14.4 / Matplotlib 3.10.7.

Run from the desired output directory. No training data or artwork is used.
"""
from pathlib import Path

import matplotlib.pyplot as plt


def main():
    scores = [.2] * 10 + [.8] * 10
    labels = [1] * 8 + [0] * 2 + [1] * 2 + [0] * 8
    groups = [list(range(10)), list(range(10, 20))]
    fine = [(sum(scores[i] for i in g)/len(g),
             sum(labels[i] for i in g)/len(g), len(g)) for g in groups]
    coarse = [(sum(scores)/20, sum(labels)/20, 20)]
    assert all(abs(x-a) < 1e-12 and abs(y-b) < 1e-12 and n == 10
               for (x, y, n), (a, b) in zip(fine, ((.2, .8), (.8, .2))))
    assert abs(coarse[0][0]-.5) < 1e-12 and coarse[0][1] == .5
    plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11})
    fig, axes = plt.subplots(1, 2, figsize=(10, 4.9), dpi=135,
                             facecolor="white", sharey=True)
    for ax, data, title in zip(axes, (fine, coarse),
                               ("Two bins: [0, 0.5), [0.5, 1]", "One bin: [0, 1]")):
        ax.plot([0, 1], [0, 1], "--", color="#7c8991", lw=1.3)
        for x, y, n in data:
            ax.scatter([x], [y], s=100 if n == 10 else 180,
                       color="#a34329" if n == 10 else "#166783", zorder=3)
            ax.annotate(f"({x:.1f}, {y:.1f}), n={n}", (x, y),
                        xytext=(0, 12), textcoords="offset points", ha="center")
        ax.set(xlim=(-.03, 1.03), ylim=(-.03, 1.03), title=title,
               xlabel="Mean predicted positive probability")
        ax.set_aspect("equal")
        ax.grid(alpha=.15)
    axes[0].set_ylabel("Observed positive fraction")
    fig.suptitle("Same 20 predictions; different binning", fontsize=17, weight="bold")
    fig.text(.5, .025, "Synthetic outcomes: 8/10 positives at score 0.2; 2/10 at score 0.8. No population guarantee is inferred.",
             ha="center", fontsize=10)
    fig.tight_layout(rect=(0, .07, 1, .91))
    fig.savefig(Path.cwd() / (Path(__file__).stem + ".png"), facecolor="white")
    plt.close(fig)


if __name__ == "__main__":
    main()