关于「倒 U:阅读预算下的块长与召回」,《分块与混合配比》给出的定量结论是什么?

结论:事实跨度为 12 时,最优块长是 48,是跨度的 4.00 倍;跨度为 24 时,最优块长仍是 48,是跨度的 2.00 倍;跨度为 48 时,最优块长移到 96,同样是 2.00 倍。跨度为 48 时,块长 8 的召回是 0.000:任何一块都不可能装下八成证据,排得再准也没有用。跨度为 12 时,小块的处境好得多,块长 8 的召回有 0.681,但仍低于块长 48 的 0.869。

数据口径:书中参数为示意;全部计算由书中程序生成;程序不读写任何文件,不连接任何线上知识库;本书是非个人化经营建议,任何线上知识库参数的改动须经离线评测并由主事者人工复核放行。 第五章扫描混合权重:编号型查询最优 0.0,改写型最优 1.0,示意基线 0.7 / 0.3 在全改写型时遗憾 0.098;按类型分流在判别出错率为 0.10 时留出召回 0.928,已低于调好的全局权重 0.938。

English version

对照数据表

下表数字由本章 Python 程序原样打印,可重新运行复现。 程序只用 Python 标准库与结论吻合的打印值: 0.000, 0.681, 0.869, 12, 2.00, 24

span8162432486496192bestw/span
120.6810.8140.8360.8410.8690.8390.7590.561484.00
240.1560.5120.5780.6220.7330.7060.6940.528482.00
480.0000.0530.2440.2410.4300.5810.6200.544962.00
查看完整验算程序
# Chapter 1. p12: the best chunk length moves with the answer span.
# Model: as p11 but 160 documents; the fact span is 12, 24 or 48 tokens.
# Fixed budget of 192 tokens, hybrid 0.7 / 0.3, no overlap. For each
# span: recall at each chunk length and the argmax. Illustrative.
import bisect
import heapq
import math
import random
from collections import Counter, defaultdict

SEED = 20261113
random.seed(SEED)  # 全局种子;各处另用独立的 Random 实例


def corpus(ndoc=240, doclen=300, span=24, seed=SEED):
    # 合成语料:每篇一个主题,两段事实;事实 = 一个编号 + 四个要点概念
    # 每个概念有三种写法(x、y、z),模拟同义改写
    rng = random.Random(seed)
    w = [1.0 / i for i in range(1, 301)]
    cdf = [math.fsum(w[:i + 1]) / math.fsum(w) for i in range(300)]
    cdf[-1] = 1.0
    docs, facts = [], []
    for d in range(ndoc):
        t, toks = d % 10, []
        for _ in range(doclen):
            if rng.random() < 0.5:
                j = rng.randrange(45)
                c = ("t%02d%02d" % (t, j) if j < 25
                     else "k%02d%02d" % (t, j - 25))
            else:
                c = "b%03d" % bisect.bisect_left(cdf, rng.random())
            toks.append(c + "xyz"[rng.randrange(3)])
        while True:
            a1 = rng.randrange(doclen - span + 1)
            a2 = rng.randrange(doclen - span + 1)
            if abs(a1 - a2) >= span:
                break
        rep = 2 if span >= 9 else 1
        for a in sorted((a1, a2)):
            keys = rng.sample(range(20), 4)
            forms = [rng.randrange(3) for _ in keys]
            pos = rng.sample(range(a, a + span), 1 + 4 * rep)
            toks[pos[0]] = "id%04d" % len(facts)
            for j, k in enumerate(keys):
                for p in pos[1 + rep * j:1 + rep * (j + 1)]:
                    toks[p] = "k%02d%02d%s" % (t, k, "xyz"[forms[j]])
            facts.append((d, a, a + span, t, keys, forms))
        docs.append(toks)
    return docs, facts


def queries(facts, keep=0.25, seed=SEED + 1):
    # 编号型:编号 + 两个原文写法的要点;改写型:四个要点,多数换写法
    rng = random.Random(seed)
    out = []
    for n, (d, a, b, t, keys, forms) in enumerate(facts):
        kw = ["id%04d" % n]
        for j in rng.sample(range(4), 2):
            kw.append("k%02d%02d%s" % (t, keys[j], "xyz"[forms[j]]))
        pa = []
        for j in range(4):
            f = forms[j]
            if rng.random() > keep:
                f = (f + 1 + rng.randrange(2)) % 3
            pa.append("k%02d%02d%s" % (t, keys[j], "xyz"[f]))
        out.append(("id", kw, d, a, b))
        out.append(("para", pa, d, a, b))
    return out


def concept(tok):
    # 向量侧的示意:同义写法并为一个概念,所有编号并为一个概念
    return "ID" if tok.startswith("id") else tok[:-1]


def chunks(docs, w, ov):
    step, out = max(1, int(round(w * (1 - ov)))), []
    for d, toks in enumerate(docs):
        s = 0
        while True:
            out.append((d, s, min(s + w, len(toks))))
            if s + w >= len(toks):
                break
            s += step
    return out


class Index:
    def __init__(self, docs, ch, k1=1.2, b=0.75):
        self.ch, self.k1, self.b, self.n = ch, k1, b, len(ch)
        self.post, self.cpost = defaultdict(list), defaultdict(list)
        self.len = [e - s for _, s, e in ch]
        self.avg = math.fsum(self.len) / self.n
        ctf, cdf = [], Counter()
        for i, (d, s, e) in enumerate(ch):
            for t, f in Counter(docs[d][s:e]).items():
                self.post[t].append((i, f))
            ctf.append(Counter(concept(t) for t in docs[d][s:e]))
            cdf.update(ctf[-1].keys())
        self.idf = {c: math.log(self.n / v) for c, v in cdf.items()}
        self.norm = []
        for i, cf in enumerate(ctf):
            xs = [(1 + math.log(v)) * self.idf[c] for c, v in cf.items()]
            for (c, _), x in zip(cf.items(), xs):
                self.cpost[c].append((i, x))
            self.norm.append(math.sqrt(math.fsum(x * x for x in xs)) or 1.0)

    def bm25(self, q):
        sc = defaultdict(list)
        for t in q:
            p = self.post.get(t, [])
            idf = math.log(1 + (self.n - len(p) + 0.5) / (len(p) + 0.5))
            for i, f in p:
                den = f + self.k1 * (1 - self.b + self.b * self.len[i]
                                     / self.avg)
                sc[i].append(idf * f * (self.k1 + 1) / den)
        return {i: math.fsum(v) for i, v in sc.items()}

    def cos(self, q):
        qc = Counter(concept(t) for t in q)
        qw = {c: (1 + math.log(v)) * self.idf[c]
              for c, v in qc.items() if c in self.idf}
        qn = math.sqrt(math.fsum(x * x for x in qw.values())) or 1.0
        sc = defaultdict(list)
        for c, x in qw.items():
            for i, y in self.cpost[c]:
                sc[i].append(x * y)
        return {i: math.fsum(v) / (qn * self.norm[i])
                for i, v in sc.items()}


def scale(s):
    # 理论最小值为零的极大值归一:分数 / 本查询最高分
    hi = max(s.values(), default=0.0) or 1.0
    return {i: v / hi for i, v in s.items()}


def pair(ix, q):
    return scale(ix.cos(q)), scale(ix.bm25(q))


def rank(a, b, alpha, k):
    # 混合分 = alpha * 向量侧 + (1 - alpha) * 词频侧
    sc = [(-(alpha * a.get(i, 0.0) + (1 - alpha) * b.get(i, 0.0)), i)
          for i in sorted(set(a) | set(b))]
    return [i for _, i in heapq.nsmallest(k, sc)]


def hit(ix, got, d, a, b, need=0.8):
    # 命中:取回的块合起来覆盖事实段至少八成
    cov = set()
    for i in got:
        dd, s, e = ix.ch[i]
        if dd == d:
            cov.update(range(max(s, a), min(e, b)))
    return len(cov) >= need * (b - a)


ALPHA, BUDGET = 0.7, 192
SIZES = (8, 16, 24, 32, 48, 64, 96, 192)
print("span " + " ".join("%6d" % w for w in SIZES) + "   best  w/span")
for span in (12, 24, 48):
    docs, facts = corpus(ndoc=160, span=span)
    qs = queries(facts)
    row = []
    for w in SIZES:
        ix = Index(docs, chunks(docs, w, 0.0))
        kb = max(1, BUDGET // w)
        h = 0
        for typ, q, d, a, b in qs:
            h += hit(ix, rank(*pair(ix, q), ALPHA, kb), d, a, b)
        row.append(h / len(qs))
    j = row.index(max(row))
    print("%4d " % span + " ".join("%6.3f" % r for r in row)
          + "   %4d  %6.2f" % (SIZES[j], SIZES[j] / span))

唯一数据源与引用

范雅福(Fan, Y.). (2026). 《知识库分块边界网格寻优、重叠率与向量词频混合检索配比母典》第 1 章「倒 U:阅读预算下的块长与召回」(版本 1.0.0). 5D Library. DOI 待发布

DOI 待发布 · 专著页正式发布后上线(本站暂无专著页) · 5DL-B-261 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026261,
  author    = {Fan, Yafu},
  title     = {Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

本书其他章节结论