分词、BPE 与 Embedding：文本如何变成向量

import java.util.*;
public class Main {
  record Pair(String left,String right) {}
  static List<String> chars(String word){
    return word.codePoints().mapToObj(c->new String(Character.toChars(c))).toList();
  }
  static int compare(Pair a,Pair b){
    int c=a.left().compareTo(b.left());return c!=0?c:a.right().compareTo(b.right());
  }
  static Pair best(List<List<String>> words){
    Map<Pair,Integer> counts=new HashMap<>();
    for(List<String> word:words)for(int i=0;i+1<word.size();i++)
      counts.merge(new Pair(word.get(i),word.get(i+1)),1,Math::addExact); // 统计包含重叠相邻位置
    Pair answer=null;int maximum=0;
    for(var entry:counts.entrySet()){
      if(entry.getValue()>maximum||(entry.getValue()==maximum&&compare(entry.getKey(),answer)<0)){
        answer=entry.getKey();maximum=entry.getValue();
      }
    }
    return answer;
  }
  static List<String> merge(List<String> word,Pair pair){
    List<String> out=new ArrayList<>();
    for(int i=0;i<word.size();){
      if(pair!=null&&i+1<word.size()&&word.get(i).equals(pair.left())&&word.get(i+1).equals(pair.right())){
        out.add(word.get(i)+word.get(i+1));i+=2; // 本轮只做非重叠合并
      }else{out.add(word.get(i));i++;}
    }
    return List.copyOf(out); // 返回独立结果，不在遍历期间修改输入
  }
  static void check(boolean v){if(!v)throw new AssertionError();}
  public static void main(String[] args){
    check(best(List.of(chars("ab"),chars("ac"))).equals(new Pair("a","b")));
    check(merge(chars("aaaa"),new Pair("a","a")).equals(List.of("aa","aa")));
    check(best(List.of(List.of()))==null);
    check(merge(chars("a"),new Pair("a","a")).equals(List.of("a")));
    check(chars("😊😊").size()==2&&merge(chars("😊😊"),new Pair("😊","😊")).size()==1);
    List<String> merged=merge(chars("abab"),new Pair("a","b"));
    check(String.join("",merged).equals("abab"));
    System.out.println("6 BPE checks passed");
  }
}