Huber 已经限制了梯度，为什么一个大输入仍能把更新放大？

"""Finite scalar Huber examples; standard library only; no model training."""
import math

CHECKS = []

def finite(value):
    if isinstance(value, bool) or not isinstance(value, (int, float)):
        raise ValueError("finite real number required")
    try:
        result = float(value)
    except OverflowError as exc:
        raise ValueError("number outside float range") from exc
    if not math.isfinite(result):
        raise ValueError("finite real number required")
    return result

def huber(residual, delta):
    r, d = finite(residual), finite(delta)
    if d <= 0:
        raise ValueError("delta must be positive")
    loss = 0.5 * r * r if abs(r) <= d else d * (abs(r) - 0.5 * d)
    derivative = max(-d, min(d, r))
    if not math.isfinite(loss):
        raise ValueError("loss outside finite range")
    return loss, derivative

def sample(x, y, w, delta):
    x, y, w = finite(x), finite(y), finite(w)
    residual = w * x - y
    loss, derivative = huber(residual, delta)
    gradient = x * derivative
    if not math.isfinite(gradient):
        raise ValueError("gradient outside finite range")
    return residual, loss, derivative, gradient

def check(name, ok):
    if not ok:
        raise AssertionError(name)
    CHECKS.append(name)

def close(a, b):
    return math.isclose(a, b, rel_tol=1e-7, abs_tol=1e-7)

def main():
    rows = [('small_input', 1., -2., 0., 1.),
            ('large_input', 100., -2., 0., 1.),
            ('small_residual_large_input', 100., -0.5, 0., 1.),
            ('negative_input', -100., -2., 0., 1.)]
    for name, x, y, w, delta in rows:
        r, loss, derivative, gradient = sample(x, y, w, delta)
        h = 1e-6
        numeric = (sample(x, y, w+h, delta)[1] - sample(x, y, w-h, delta)[1]) / (2*h)
        check(name + ' finite difference', close(gradient, numeric))
        check(name + ' residual derivative bound', abs(derivative) <= delta)
        print(f'{name}: r={r:.2f}; loss={loss:.2f}; dL_dr={derivative:.2f}; dL_dw={gradient:.2f}')

    # Same two samples and sum reduction, but three different operation orders.
    xs, residuals, delta = [100., 1.], [2., -3.], 1.
    huber_sum = sum(x*huber(r, delta)[1] for x, r in zip(xs, residuals))
    l2_sum = sum(x*r for x, r in zip(xs, residuals))
    clip_after_sum = max(-delta, min(delta, l2_sum))
    clip_each_parameter = sum(max(-delta, min(delta, x*r)) for x, r in zip(xs, residuals))
    check('three operations differ', (huber_sum, clip_after_sum, clip_each_parameter) == (99., 1., 0.))
    print(f'two_samples_sum: huber={huber_sum:.2f}; half_squared={l2_sum:.2f}; clip_total={clip_after_sum:.2f}; clip_each_parameter={clip_each_parameter:.2f}')

    for d in [0.5, 1., 2.]:
        for r in [-4., -2., -0.5, 0., 0.5, 2., 4.]:
            loss, derivative = huber(r, d)
            check('even loss', close(loss, huber(-r, d)[0]))
            check('odd derivative', close(derivative, -huber(-r, d)[1]))
            check('residual derivative bounded', abs(derivative) <= d)
        check('boundary loss', close(huber(d, d)[0], 0.5*d*d))
        check('boundary derivative', close(huber(-d, d)[1], -d))
    # Coordinate change: x2=100*x, w2=w/100 gives the same prediction.
    first, second = sample(1., 0., 2., 1.), sample(100., 0., 0.02, 1.)
    check('same prediction and loss', first[:3] == second[:3])
    check('parameter-coordinate derivative changes', close(second[3], 100*first[3]))
    print(f'same_prediction_new_coordinate: loss={first[1]:.2f}; gradient_before={first[3]:.2f}; gradient_after={second[3]:.2f}')
    invalid = [(0., 0.), (0., -1.), (float('nan'), 1.), (1., float('inf')), (True, 1.), ('2', 1.), (10**1000, 1.)]
    for args in invalid:
        try:
            huber(*args)
        except ValueError:
            check('reject invalid Huber input', True)
        else:
            raise AssertionError('invalid input accepted')
    print(f'PASS checks={len(CHECKS)}; reduction=sum; reference=chain_rule+finite_difference')

if __name__ == '__main__':
    main()