Dropout把平均值保住了，关掉后输出就一样吗？

"""Enumerate independent Bernoulli masks; all probability and loss math is exact."""
import csv
import json
from fractions import Fraction as F
from itertools import product
from pathlib import Path

ROOT = Path(__file__).resolve().parent
CASES = [
    {"id": "cancel_p0", "x": ["1", "-1"], "drop_p": "0"},
    {"id": "cancel_p25", "x": ["1", "-1"], "drop_p": "1/4"},
    {"id": "cancel_p50", "x": ["1", "-1"], "drop_p": "1/2"},
    {"id": "cancel_p75", "x": ["1", "-1"], "drop_p": "3/4"},
    {"id": "positive_p50", "x": ["1", "1"], "drop_p": "1/2"},
]


def relu(value):
    return max(F(0), value)


def main():
    rows, reports = [], []
    for case in CASES:
        x = list(map(F, case["x"]))
        p = F(case["drop_p"])
        assert len(x) == 2 and 0 <= p < 1  # Toy excludes p=1 because it divides by 1-p.
        q = 1 - p
        masks = []
        for mask in product((0, 1), repeat=2):
            probability = F(1)
            for bit in mask:
                probability *= q if bit else p
            dropped = [value * bit / q for value, bit in zip(x, mask)]
            preactivation = sum(dropped, F(0))
            output = relu(preactivation)
            masks.append((probability, dropped, preactivation, output))
            rows.append({"case": case["id"], "drop_p": str(p), "keep_q": str(q),
                         "mask": "".join(map(str, mask)), "probability": str(probability),
                         "dropped_0": str(dropped[0]), "dropped_1": str(dropped[1]),
                         "preactivation": str(preactivation), "relu_output": str(output),
                         "squared_loss_target0": str(output ** 2),
                         "squared_loss_target1": str((output - 1) ** 2)})
        assert sum(probability for probability, *_ in masks) == 1
        mean_dropped = [sum(prob * values[i] for prob, values, _, _ in masks) for i in range(2)]
        mean_pre = sum(prob * pre for prob, _, pre, _ in masks)
        mean_output = sum(prob * out for prob, _, _, out in masks)
        second_moment = sum(prob * out ** 2 for prob, _, _, out in masks)
        variance = second_moment - mean_output ** 2
        eval_pre, eval_output = sum(x, F(0)), relu(sum(x, F(0)))
        assert mean_dropped == x and mean_pre == eval_pre
        losses = []
        for target in (F(0), F(1)):
            mean_loss = sum(prob * (out - target) ** 2 for prob, _, _, out in masks)
            loss_at_mean = (mean_output - target) ** 2
            assert mean_loss == loss_at_mean + variance
            losses.append({"target": str(target), "mean_training_loss": str(mean_loss),
                           "loss_of_mean_training_output": str(loss_at_mean),
                           "eval_loss": str((eval_output - target) ** 2)})
        reports.append({"case": case["id"], "drop_p": str(p), "keep_q": str(q),
                        "mean_dropped": list(map(str, mean_dropped)), "mean_preactivation": str(mean_pre),
                        "mean_training_output": str(mean_output), "eval_output": str(eval_output),
                        "output_variance": str(variance), "losses": losses})
    assert reports[2]["mean_training_output"] == "1/2"
    assert reports[2]["eval_output"] == "0"
    assert reports[2]["losses"][0]["mean_training_loss"] == "1"
    assert reports[4]["mean_training_output"] == reports[4]["eval_output"] == "2"
    assert reports[4]["losses"][0]["mean_training_loss"] == "6"
    inputs = {"mask_distribution": "two independent Bernoulli keep(q), q=1-drop_p",
              "training_transform": "x_i * mask_i / q", "evaluation_transform": "identity",
              "network": "ReLU(sum(transformed_x)); no parameters change", "cases": CASES}
    report = {"scope": "Exact enumeration of four masks per case; no random sampling or training",
              "toy_p_range": "0 <= drop_p < 1; does not cover framework p=1 boundary",
              "cases": reports, "mask_rows": len(rows), "loss_summaries": 2 * len(reports)}
    (ROOT / "dropout-inputs.json").write_text(json.dumps(inputs, indent=2) + "\n", encoding="utf-8")
    (ROOT / "dropout-report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
    with (ROOT / "dropout-masks.csv").open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)
    print(f"PASS dropout_expectation: {len(CASES)} cases, {len(rows)} mask rows, {report['loss_summaries']} loss summaries")


if __name__ == "__main__":
    main()