What does Active Learning for Intent Labels conclude, in numbers, about “Sampling Bias: Picked Messages No Longer Stand for the Traffic”?

Answer: Chapter 4 measures the bias: the margin-picked labeled set lies at a total variation distance of 0.208 from the traffic, and a model graded on the messages it picked scores only 0.463 against a true accuracy of 0.876; in a one-dimensional example uncertainty sampling stays at the wrong threshold with probability 0.513 even after three hundred labels, against 0.068 for random sampling.

Scope of the numbers: Intent priors, word probabilities and costs are illustrative; all calculations are generated by the book's programs; the book is general business analysis, not individualised business advice. Marked boxes separate published evidence, runnable Python calculations (23 programs), classical sources and the author's hypotheses.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 0.463, 0.876

test shares   0.619 0.224 0.108 0.050
random 640: accuracy 0.867  ECE 0.015
  mean predicted prior 0.589 0.247 0.113 0.051
  480-label model graded on next 160 picks 0.869, on test 0.858
margin 640: accuracy 0.879  ECE 0.022
  mean predicted prior 0.602 0.235 0.107 0.055
  480-label model graded on next 160 picks 0.463, on test 0.876
Show the full program
# Chapter 4. p42: calibration and self-graded accuracy under biased labels.
# 同样 640 条标注,随机集与间隔集训出的模型:测试准确率、
# 期望校准误差 ECE(按最大概率分 10 档)、平均预测概率估计的意图先验;
# 以及「用刚挑来的那批当考卷」自评准确率与真实准确率之差(种子 11)。
import math
import random

WORDS = (
    "where track tracking ship shipped late delay courier parcel arrive "
    "refund money back charge charged return cancel paid reimburse "
    "credit size swap small large color exchange fit tight wrong "
    "replace agent human person manager complain angry supervisor "
    "call speak rude order please hi hello thanks item today help "
    "not my the is it can you want").split()
INTENTS = ("track", "refund", "exchange", "human")
V, K = len(WORDS), 4
PRIOR = (0.62, 0.22, 0.11, 0.05)   # 意图先验(示意)
PS, PC = 0.40, 0.10                # 本意图词、易混意图词的概率(示意)
CONF = (1, 0, 1, 1)                # 每个意图最易混的另一意图
ZW = [1.0 / (r + 1) for r in range(10)]   # 意图词内部的齐普夫权重


def make_doc(rng, k):
    # 一条去标识化客服消息:3 到 8 个词,表示为(词号, 次数)
    cnt = {}
    for _ in range(rng.randint(3, 8)):
        u = rng.random()
        if u < PS:
            j = rng.choices(range(10 * k, 10 * k + 10), ZW)[0]
        elif u < PS + PC:
            c = CONF[k]
            j = rng.choices(range(10 * c, 10 * c + 10), ZW)[0]
        else:
            j = rng.randrange(40, V)
        cnt[j] = cnt.get(j, 0) + 1
    return tuple(sorted(cnt.items()))


def make_data(seed, n):
    rng = random.Random(seed)
    xs, ys = [], []
    for _ in range(n):
        k = rng.choices(range(K), PRIOR)[0]
        xs.append(make_doc(rng, k))
        ys.append(k)
    return xs, ys


def probs(W, x):
    # 多类逻辑回归(softmax):每类得分 = 偏置 + 词权重乘计数
    s = [math.fsum([W[k][V]] + [W[k][j] * c for j, c in x])
         for k in range(K)]
    m = max(s)
    e = [math.exp(v - m) for v in s]
    z = math.fsum(e)
    return [v / z for v in e]


def sgd(W, xs, ys, idx, epochs, lr0, rng, lam=0.002):
    # 随机梯度下降:每步只更新一次权重(单步状态更新)
    order = list(idx)
    for ep in range(epochs):
        lr = lr0 / (1 + ep)
        rng.shuffle(order)
        for i in order:
            p = probs(W, xs[i])
            for k in range(K):
                g = p[k] - (1.0 if ys[i] == k else 0.0)
                row = W[k]
                for j, c in xs[i]:
                    row[j] -= lr * (g * c + lam * row[j])
                row[V] -= lr * g
    return W


def fit(xs, ys, idx):
    W = [[0.0] * (V + 1) for _ in range(K)]
    return sgd(W, xs, ys, idx, 10, 0.3, random.Random(7))


def argmax(p):
    return p.index(max(p))


def accuracy(W, xs, ys):
    hit = 0
    for x, y in zip(xs, ys):
        hit += argmax(probs(W, x)) == y
    return hit / len(xs)


def score(strategy, p):
    # 越小越优先标注
    a = sorted(p, reverse=True)
    if strategy == "margin":
        return a[0] - a[1]
    if strategy == "least":
        return a[0]
    if strategy == "entropy":
        return math.fsum(v * math.log(v) for v in p if v > 0)
    return -a[0]                     # "easy": 最有把握的先标


GRID = (20, 30, 40, 60, 80, 100, 120, 160, 200, 240, 320, 400, 480,
        640, 800, 960, 1280, 1600, 1920, 2560, 3200)


def run(strategy, seed, pool, test, top):
    # 返回 {标注数: 测试准确率} 与已标注下标(按标注先后)
    px, py = pool
    rng = random.Random(seed)
    order = list(range(len(px)))
    rng.shuffle(order)
    lab = order[:GRID[0]]
    taken = set(lab)
    W = fit(px, py, lab)
    curve = {GRID[0]: accuracy(W, *test)}
    for g in GRID[1:]:
        if g > top:
            break
        need = g - len(lab)
        if strategy == "random":
            cand = order[len(lab):g]
        else:
            sc = sorted((score(strategy, probs(W, px[i])), i)
                        for i in range(len(px)) if i not in taken)
            cand = [i for _, i in sc[:need]]
        taken.update(cand)
        lab = lab + cand
        W = fit(px, py, lab)
        curve[g] = accuracy(W, *test)
    return curve, lab, W


def first_reach(curve, target):
    for g in sorted(curve):
        if curve[g] >= target:
            return g
    return None


def ece(W, xs, ys, bins=10):
    cells = [[] for _ in range(bins)]
    for x, y in zip(xs, ys):
        p = probs(W, x)
        b = min(bins - 1, int(max(p) * bins))
        cells[b].append((max(p), argmax(p) == y))
    gaps = []
    for c in cells:
        if c:
            conf = math.fsum(v for v, _ in c) / len(c)
            acc = [h for _, h in c].count(True) / len(c)
            gaps.append(len(c) * abs(conf - acc))
    return math.fsum(gaps) / len(xs)


def mean_prob(W, xs):
    P = [probs(W, x) for x in xs]
    return [math.fsum(p[k] for p in P) / len(P) for k in range(K)]


pool, test = make_data(1, 6000), make_data(2, 3000)
px, py = pool
tx, ty = test
truth = [ty.count(k) / len(ty) for k in range(K)]
print("test shares   " + " ".join("{:.3f}".format(v) for v in truth))
for s in ("random", "margin"):
    _, lab, _ = run(s, 11, pool, test, 640)
    W = fit(px, py, lab)
    mp = mean_prob(W, tx)
    print("{:<6s} 640: accuracy {:.3f}  ECE {:.3f}".format(
        s, accuracy(W, *test), ece(W, *test)))
    print("  mean predicted prior " + " ".join(
        "{:.3f}".format(v) for v in mp))
    W0 = fit(px, py, lab[:480])
    new = lab[480:640]
    self_acc = accuracy(W0, [px[i] for i in new], [py[i] for i in new])
    print("  480-label model graded on next 160 picks {:.3f}, on test "
          "{:.3f}".format(self_acc, accuracy(W0, *test)))

Single source and citation

Fan, Y. (2026). Active Learning and Uncertainty Sampling: the Fewest Human Labels a Customer-Intent Classifier Needs to Reach Its Target (Version 1.0.0), Chapter 4: Sampling Bias: Picked Messages No Longer Stand for the Traffic. 5D Library. DOI pending

DOI pending · monograph page goes live after publication · 5DL-B-312 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026312,
  author    = {Fan, Yafu},
  title     = {Active Learning and Uncertainty Sampling: the Fewest Human Labels a Customer-Intent Classifier Needs to Reach Its Target},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

Other chapter answers in this book