 codex/conservative-variance-low-coverage.py

"""An expectation-correct variance estimate need not justify Wald coverage."""
from math import erf, sqrt
from pathlib import Path

import matplotlib.pyplot as plt


def main():
    epsilon = 0.01
    estimates = [0, 1 / epsilon]
    probabilities = [1 - epsilon, epsilon]
    assert sum(v * p for v, p in zip(estimates, probabilities)) == 1
    # erf rounds the extremely small omitted normal tail to zero here.
    coverage = epsilon * erf(1.96 / sqrt(2 * epsilon))
    assert 0 <= coverage <= epsilon

    fig, (mass, intervals) = plt.subplots(1, 2, figsize=(9.6, 4.2), dpi=100,
                                        layout="constrained")
    fig.set_facecolor("white")
    bars = mass.bar([0, 1], probabilities, color=["#437fa3", "#bb6d37"], width=0.6)
    mass.bar_label(bars, labels=["99%", "1%"], padding=4)
    mass.set(xticks=[0, 1], xticklabels=["Variance estimate = 0", "Variance estimate = 100"],
             ylim=(0, 1.12), ylabel="Probability",
             title="Mean estimate = true variance = 1")
    bars = intervals.bar([0, 1], [0.95, coverage],
                         color=["#899399", "#bb6d37"], width=0.6)
    intervals.bar_label(bars, labels=["95%", "≤1%"], padding=4)
    intervals.set(xticks=[0, 1], xticklabels=["Nominal coverage", "Actual coverage"],
                  ylim=(0, 1.12), ylabel="Coverage probability",
                  title="An expectation bound cannot ensure coverage")
    for ax in [mass, intervals]:
        ax.grid(axis="y", alpha=0.15)
        ax.set_axisbelow(True)
    fig.savefig(Path(__file__).with_suffix(".png").name, facecolor="white")
    plt.close(fig)


if __name__ == "__main__":
    main()