预训练语料：清洗、去重、来源与污染检查

import java.util.*;
public class Main {
    static int root(int[] p, int x) {
        // 路径压缩保持家族代表不变
        return p[x] == x ? x : (p[x] = root(p, p[x]));
    }
    static int[] group(List<Set<Integer>> docs, double threshold) {
        if (!Double.isFinite(threshold) || threshold <= 0 || threshold > 1)
            throw new IllegalArgumentException("threshold");
        int n = docs.size(); int[] p = new int[n], rank = new int[n], minimum = new int[n];
        for (int i=0;i<n;i++) {
            if(docs.get(i)==null || docs.get(i).isEmpty())
                throw new IllegalArgumentException("empty");
            p[i]=i; minimum[i]=i;
        }
        for(int i=0;i<n;i++) for(int j=i+1;j<n;j++) {
            int common=0;
            for(int token:docs.get(i)) if(docs.get(j).contains(token)) common++;
            long union=(long)docs.get(i).size()+docs.get(j).size()-common;
            // 除法转浮点前用long形成并集大小，避免整数相加溢出
            if((double)common/union >= threshold) {
                int a=root(p,i), b=root(p,j);
                if(a!=b){
                    if(rank[a]<rank[b]){int tmp=a;a=b;b=tmp;}
                    p[b]=a; minimum[a]=Math.min(minimum[a],minimum[b]);
                    if(rank[a]==rank[b])rank[a]++;
                }
            }
        }
        // 最小代表与树根分离，避免牺牲按秩归并的复杂度
        int[] labels=new int[n];
        for(int i=0;i<n;i++) labels[i]=minimum[root(p,i)];
        return labels;
    }
    static void check(boolean ok){if(!ok) throw new AssertionError();}
    public static void main(String[] args){
        var chain=List.of(Set.of(1,2),Set.of(2,3),Set.of(3,4));
        check(Arrays.equals(group(chain,1.0/3),new int[]{0,0,0}));
        check(Arrays.equals(group(chain,0.5),new int[]{0,1,2}));
        check(Arrays.equals(group(List.of(Set.of(1),Set.of(1)),1),new int[]{0,0}));
        check(group(List.of(),1).length==0);
        try {group(chain,Double.NaN); throw new AssertionError();}
        catch(IllegalArgumentException expected){}
        try {group(List.of(Set.of()),1); throw new AssertionError();}
        catch(IllegalArgumentException expected){}
        System.out.println("6 duplicate family checks passed");
    }
}