学习率预热、调度与梯度裁剪

import math
def clip_reference(gradients, maximum):
    norm = math.sqrt(sum(x*x for row in gradients for x in row))
    if not math.isfinite(norm):
        raise ValueError("nonfinite gradient")  # 异常数值单独处理
    coefficient = min(1.0, maximum / (norm + 1e-6))
    clipped = [[x * coefficient for x in row] for row in gradients]
    return norm, clipped  # 返回旧范数，全部参数使用同一缩放系数
# 此函数返回副本；PyTorch真实接口原地修改Parameter.grad

public class Main {
  static double rate(int u,int total,int warm,double peak,double floor){
    if(total<=0||warm<0||warm>=total||u<0||u>total||!Double.isFinite(peak)||
       !Double.isFinite(floor)||floor<0||peak<floor)throw new IllegalArgumentException();
    if(warm>0&&u<=warm)return peak*u/warm; // 预热端点达到峰值
    return floor+(peak-floor)*(1+Math.cos(Math.PI*(u-warm)/(total-warm)))/2;
  }
  static double[] clip(double[] gradient,double cap){
    if(!Double.isFinite(cap)||cap<0)throw new IllegalArgumentException();
    double norm=0;
    for(double x:gradient){
      if(!Double.isFinite(x))throw new IllegalArgumentException();
      norm=Math.hypot(norm,x); // 避免直接平方的大中间值
    }
    if(!Double.isFinite(norm))throw new ArithmeticException("norm range");
    double coefficient=norm==0?1:Math.min(1,cap/norm);
    double[] out=new double[gradient.length];
    for(int i=0;i<out.length;i++)out[i]=gradient[i]*coefficient; // 全向量统一比例
    return out;
  }
  static void close(double a,double b){if(Math.abs(a-b)>1e-9)throw new AssertionError();}
  public static void main(String[] args){
    close(rate(0,6,2,1,.1),0);close(rate(2,6,2,1,.1),1);close(rate(6,6,2,1,.1),.1);
    double[] c=clip(new double[]{3,4},2);close(c[0],1.2);close(c[1],1.6);
    close(clip(new double[]{3,4},10)[0],3);
    close(clip(new double[]{0,0},0)[0],0);
    close(rate(0,6,0,1,.1),1);
    boolean bad=false;try{rate(1,2,2,1,0);}catch(IllegalArgumentException e){bad=true;}
    if(!bad)throw new AssertionError();
    System.out.println("8 schedule clipping checks passed");
  }
}