关于「重要性加权与先验校正:把偏差算回来」,《主动学习与客服意图标注》给出的定量结论是什么?

结论:第五章以已知入选概率抽样并按霍维茨与汤普森的倒数加权,把一个真实准确率 0.809 的模型的打分从直接平均的 0.657 校正回真值,并说明加权训练与只校正先验在有限样本下可能更差。

数据口径:书中意图先验、词概率与成本均为示意参数;全部计算由书中程序生成;本书属于非个人化经营建议。 第一章在合成的四意图客服数据上重算:达到目标准确率,随机抽样的中位数是 1,600 条,最小间隔采样是 640 条,后者只用前者的 0.400;全部标注训练的模型准确率为 0.893,这一倍数随目标逼近天花板而变化。

English version

对照数据表

下表数字由本章 Python 程序原样打印,可重新运行复现。 程序只用 Python 标准库与结论吻合的打印值: 0.657, 0.809

grading the 200-label model, naive: mean 0.657, SD 0.016
grading the 200-label model, Hajek: mean 0.809, SD 0.019
its true accuracy on the 5,800 unlabeled: 0.809
one draw: 826 labels in total (200 random seed + 626 sampled)
  unweighted accuracy 0.875  ECE 0.006  prior TV 0.007
  weighted   accuracy 0.852  ECE 0.048  prior TV 0.022
查看完整验算程序
# Chapter 5. p52: weighted grading and weighted training.
# 同 p51 的抽样设计(eps = 0.1,预算 600)。其一:用抽到的样本给一个
# 固定模型打分,直接平均 vs Hajek 加权 vs 真值(300 次重复)。
# 其二:用一次抽样训练模型,样本权重 1/pi 归一到均值 1、截断于 10,
# 比较不加权与加权的测试准确率、ECE 与预测先验偏差(示意数据)。
import math
import random

WORDS = (
    "where track tracking ship shipped late delay courier parcel arrive "
    "refund money back charge charged return cancel paid reimburse "
    "credit size swap small large color exchange fit tight wrong "
    "replace agent human person manager complain angry supervisor "
    "call speak rude order please hi hello thanks item today help "
    "not my the is it can you want").split()
INTENTS = ("track", "refund", "exchange", "human")
V, K = len(WORDS), 4
PRIOR = (0.62, 0.22, 0.11, 0.05)   # 意图先验(示意)
PS, PC = 0.40, 0.10                # 本意图词、易混意图词的概率(示意)
CONF = (1, 0, 1, 1)                # 每个意图最易混的另一意图
ZW = [1.0 / (r + 1) for r in range(10)]   # 意图词内部的齐普夫权重


def make_doc(rng, k):
    # 一条去标识化客服消息:3 到 8 个词,表示为(词号, 次数)
    cnt = {}
    for _ in range(rng.randint(3, 8)):
        u = rng.random()
        if u < PS:
            j = rng.choices(range(10 * k, 10 * k + 10), ZW)[0]
        elif u < PS + PC:
            c = CONF[k]
            j = rng.choices(range(10 * c, 10 * c + 10), ZW)[0]
        else:
            j = rng.randrange(40, V)
        cnt[j] = cnt.get(j, 0) + 1
    return tuple(sorted(cnt.items()))


def make_data(seed, n):
    rng = random.Random(seed)
    xs, ys = [], []
    for _ in range(n):
        k = rng.choices(range(K), PRIOR)[0]
        xs.append(make_doc(rng, k))
        ys.append(k)
    return xs, ys


def probs(W, x):
    # 多类逻辑回归(softmax):每类得分 = 偏置 + 词权重乘计数
    s = [math.fsum([W[k][V]] + [W[k][j] * c for j, c in x])
         for k in range(K)]
    m = max(s)
    e = [math.exp(v - m) for v in s]
    z = math.fsum(e)
    return [v / z for v in e]


def sgd(W, xs, ys, idx, epochs, lr0, rng, lam=0.002):
    # 随机梯度下降:每步只更新一次权重(单步状态更新)
    order = list(idx)
    for ep in range(epochs):
        lr = lr0 / (1 + ep)
        rng.shuffle(order)
        for i in order:
            p = probs(W, xs[i])
            for k in range(K):
                g = p[k] - (1.0 if ys[i] == k else 0.0)
                row = W[k]
                for j, c in xs[i]:
                    row[j] -= lr * (g * c + lam * row[j])
                row[V] -= lr * g
    return W


def fit(xs, ys, idx):
    W = [[0.0] * (V + 1) for _ in range(K)]
    return sgd(W, xs, ys, idx, 10, 0.3, random.Random(7))


def argmax(p):
    return p.index(max(p))


def accuracy(W, xs, ys):
    hit = 0
    for x, y in zip(xs, ys):
        hit += argmax(probs(W, x)) == y
    return hit / len(xs)


def wfit(xs, ys, idx, wt):
    W = [[0.0] * (V + 1) for _ in range(K)]
    rng = random.Random(7)
    order = list(range(len(idx)))
    for ep in range(10):
        lr = 0.3 / (1 + ep)
        rng.shuffle(order)
        for t in order:
            i = idx[t]
            p = probs(W, xs[i])
            for k in range(K):
                g = wt[t] * (p[k] - (1.0 if ys[i] == k else 0.0))
                for j, c in xs[i]:
                    W[k][j] -= lr * (g * c + 0.002 * W[k][j])
                W[k][V] -= lr * g
    return W


def ece(W, xs, ys):
    cells = [[] for _ in range(10)]
    for x, y in zip(xs, ys):
        p = probs(W, x)
        cells[min(9, int(max(p) * 10))].append((max(p), argmax(p) == y))
    gaps = [len(c) * abs(math.fsum(v for v, _ in c) / len(c)
                         - [h for _, h in c].count(True) / len(c))
            for c in cells if c]
    return math.fsum(gaps) / len(xs)


pool, test = make_data(1, 6000), make_data(2, 3000)
px, py = pool
rng = random.Random(11)
order = list(range(6000))
rng.shuffle(order)
W0 = fit(px, py, order[:200])
rest = order[200:]
N = len(rest)
u = []
for i in rest:
    a = sorted(probs(W0, px[i]), reverse=True)
    u.append(1 - (a[0] - a[1]))
U = math.fsum(u)
pi = [min(1.0, 600 * (0.1 / N + 0.9 * v / U)) for v in u]
hit = [argmax(probs(W0, px[i])) == py[i] for i in rest]
true_acc = hit.count(True) / N
draw = random.Random(20261113)
nv, hj = [], []
for _ in range(300):
    S = [t for t in range(N) if draw.random() < pi[t]]
    nv.append([hit[t] for t in S].count(True) / len(S))
    w = [1 / pi[t] for t in S]
    hj.append(math.fsum(w[a] for a, t in enumerate(S) if hit[t])
              / math.fsum(w))
for name, e in (("naive", nv), ("Hajek", hj)):
    m = math.fsum(e) / len(e)
    sd = math.sqrt(math.fsum((v - m) ** 2 for v in e) / (len(e) - 1))
    print("grading the 200-label model, {}: mean {:.3f}, SD {:.3f}"
          .format(name, m, sd))
print("its true accuracy on the {:,} unlabeled: {:.3f}".format(
    N, true_acc))
S = [t for t in range(N) if draw.random() < pi[t]]
idx = order[:200] + [rest[t] for t in S]
raw = [1.0] * 200 + [1 / pi[t] for t in S]
mean_w = math.fsum(raw) / len(raw)
wt = [min(10.0, v / mean_w) for v in raw]
tshare = [test[1].count(k) / len(test[1]) for k in range(K)]
print("one draw: {} labels in total ({} random seed + {} sampled)".format(
    len(idx), 200, len(S)))
for name, ww in (("unweighted", [1.0] * len(idx)), ("weighted", wt)):
    W = wfit(px, py, idx, ww)
    P = [probs(W, x) for x in test[0]]
    mp = [math.fsum(p[k] for p in P) / len(P) for k in range(K)]
    tv = 0.5 * math.fsum(abs(a - b) for a, b in zip(mp, tshare))
    print("  {:<10s} accuracy {:.3f}  ECE {:.3f}  prior TV {:.3f}".format(
        name, accuracy(W, *test), ece(W, *test), tv))

唯一数据源与引用

范雅福(Fan, Y.). (2026). 《主动学习与不确定性采样、最少人工标注让客服意图分类器达标母典》第 5 章「重要性加权与先验校正:把偏差算回来」(版本 1.0.0). 5D Library. DOI 待发布

DOI 待发布 · 专著页正式发布后上线(本站暂无专著页) · 5DL-B-312 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026312,
  author    = {Fan, Yafu},
  title     = {Active Learning and Uncertainty Sampling: the Fewest Human Labels a Customer-Intent Classifier Needs to Reach Its Target},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

本书其他章节结论