初始化、归一化与残差：让梯度走得更远

import java.util.*;
public class Main {
  static double[] norm(double[] x,double[] gamma,double[] beta,double eps){
    if(x.length==0||gamma.length!=x.length||beta.length!=x.length||!Double.isFinite(eps)||eps<=0)
      throw new IllegalArgumentException();
    double mean=0,m2=0;int n=0;
    for(double v:x){
      if(!Double.isFinite(v))throw new IllegalArgumentException();
      n++;double delta=v-mean;mean+=delta/n;
      m2+=delta*(v-mean); // 更新后的均值用于稳定平方差累积
    }
    double variance=m2/n;
    if(!Double.isFinite(variance)||variance<0)throw new ArithmeticException("range");
    double denominator=Math.sqrt(variance+eps); // 总体方差而非除以n-1
    double[] result=new double[n];
    for(int i=0;i<n;i++){
      if(!Double.isFinite(gamma[i])||!Double.isFinite(beta[i]))throw new IllegalArgumentException();
      result[i]=(x[i]-mean)/denominator*gamma[i]+beta[i];
      if(!Double.isFinite(result[i]))throw new ArithmeticException("range"); // 拒绝不可表示输出
    }
    return result;
  }
  static void check(boolean v){if(!v)throw new AssertionError();}
  static void close(double a,double b){check(Math.abs(a-b)<1e-9);}
  public static void main(String[] args){
    double[] r=norm(new double[]{1,2,3},new double[]{1,1,1},new double[3],1e-5);
    close(r[1],0);close(r[0],-r[2]);
    check(Arrays.equals(norm(new double[]{7,7},new double[]{1,1},new double[]{2,3},1e-5),new double[]{2,3}));
    close(norm(new double[]{5},new double[]{9},new double[]{4},1e-5)[0],4);
    double[] shifted=norm(new double[]{1e12+1,1e12+2,1e12+3},new double[]{1,1,1},new double[3],1e-5);
    close(shifted[0],r[0]); // 大公共偏置不应改变这里可精确表示的差值
    boolean bad=false;
    try{norm(new double[]{Double.NaN},new double[]{1},new double[]{0},1e-5);}catch(IllegalArgumentException e){bad=true;}
    check(bad);
    bad=false;try{norm(new double[0],new double[0],new double[0],1e-5);}catch(IllegalArgumentException e){bad=true;}
    check(bad);
    System.out.println("6 LayerNorm checks passed");
  }
}