关于「倒 U:阅读预算下的块长与召回」,《分块与混合配比》给出的定量结论是什么?
结论:事实跨度为 12 时,最优块长是 48,是跨度的 4.00 倍;跨度为 24 时,最优块长仍是 48,是跨度的 2.00 倍;跨度为 48 时,最优块长移到 96,同样是 2.00 倍。跨度为 48 时,块长 8 的召回是 0.000:任何一块都不可能装下八成证据,排得再准也没有用。跨度为 12 时,小块的处境好得多,块长 8 的召回有 0.681,但仍低于块长 48 的 0.869。
数据口径:书中参数为示意;全部计算由书中程序生成;程序不读写任何文件,不连接任何线上知识库;本书是非个人化经营建议,任何线上知识库参数的改动须经离线评测并由主事者人工复核放行。 第五章扫描混合权重:编号型查询最优 0.0,改写型最优 1.0,示意基线 0.7 / 0.3 在全改写型时遗憾 0.098;按类型分流在判别出错率为 0.10 时留出召回 0.928,已低于调好的全局权重 0.938。
对照数据表
下表数字由本章 Python 程序原样打印,可重新运行复现。 程序只用 Python 标准库与结论吻合的打印值: 0.000, 0.681, 0.869, 12, 2.00, 24
| span | 8 | 16 | 24 | 32 | 48 | 64 | 96 | 192 | best | w/span |
| 12 | 0.681 | 0.814 | 0.836 | 0.841 | 0.869 | 0.839 | 0.759 | 0.561 | 48 | 4.00 |
| 24 | 0.156 | 0.512 | 0.578 | 0.622 | 0.733 | 0.706 | 0.694 | 0.528 | 48 | 2.00 |
| 48 | 0.000 | 0.053 | 0.244 | 0.241 | 0.430 | 0.581 | 0.620 | 0.544 | 96 | 2.00 |
查看完整验算程序
# Chapter 1. p12: the best chunk length moves with the answer span.
# Model: as p11 but 160 documents; the fact span is 12, 24 or 48 tokens.
# Fixed budget of 192 tokens, hybrid 0.7 / 0.3, no overlap. For each
# span: recall at each chunk length and the argmax. Illustrative.
import bisect
import heapq
import math
import random
from collections import Counter, defaultdict
SEED = 20261113
random.seed(SEED) # 全局种子;各处另用独立的 Random 实例
def corpus(ndoc=240, doclen=300, span=24, seed=SEED):
# 合成语料:每篇一个主题,两段事实;事实 = 一个编号 + 四个要点概念
# 每个概念有三种写法(x、y、z),模拟同义改写
rng = random.Random(seed)
w = [1.0 / i for i in range(1, 301)]
cdf = [math.fsum(w[:i + 1]) / math.fsum(w) for i in range(300)]
cdf[-1] = 1.0
docs, facts = [], []
for d in range(ndoc):
t, toks = d % 10, []
for _ in range(doclen):
if rng.random() < 0.5:
j = rng.randrange(45)
c = ("t%02d%02d" % (t, j) if j < 25
else "k%02d%02d" % (t, j - 25))
else:
c = "b%03d" % bisect.bisect_left(cdf, rng.random())
toks.append(c + "xyz"[rng.randrange(3)])
while True:
a1 = rng.randrange(doclen - span + 1)
a2 = rng.randrange(doclen - span + 1)
if abs(a1 - a2) >= span:
break
rep = 2 if span >= 9 else 1
for a in sorted((a1, a2)):
keys = rng.sample(range(20), 4)
forms = [rng.randrange(3) for _ in keys]
pos = rng.sample(range(a, a + span), 1 + 4 * rep)
toks[pos[0]] = "id%04d" % len(facts)
for j, k in enumerate(keys):
for p in pos[1 + rep * j:1 + rep * (j + 1)]:
toks[p] = "k%02d%02d%s" % (t, k, "xyz"[forms[j]])
facts.append((d, a, a + span, t, keys, forms))
docs.append(toks)
return docs, facts
def queries(facts, keep=0.25, seed=SEED + 1):
# 编号型:编号 + 两个原文写法的要点;改写型:四个要点,多数换写法
rng = random.Random(seed)
out = []
for n, (d, a, b, t, keys, forms) in enumerate(facts):
kw = ["id%04d" % n]
for j in rng.sample(range(4), 2):
kw.append("k%02d%02d%s" % (t, keys[j], "xyz"[forms[j]]))
pa = []
for j in range(4):
f = forms[j]
if rng.random() > keep:
f = (f + 1 + rng.randrange(2)) % 3
pa.append("k%02d%02d%s" % (t, keys[j], "xyz"[f]))
out.append(("id", kw, d, a, b))
out.append(("para", pa, d, a, b))
return out
def concept(tok):
# 向量侧的示意:同义写法并为一个概念,所有编号并为一个概念
return "ID" if tok.startswith("id") else tok[:-1]
def chunks(docs, w, ov):
step, out = max(1, int(round(w * (1 - ov)))), []
for d, toks in enumerate(docs):
s = 0
while True:
out.append((d, s, min(s + w, len(toks))))
if s + w >= len(toks):
break
s += step
return out
class Index:
def __init__(self, docs, ch, k1=1.2, b=0.75):
self.ch, self.k1, self.b, self.n = ch, k1, b, len(ch)
self.post, self.cpost = defaultdict(list), defaultdict(list)
self.len = [e - s for _, s, e in ch]
self.avg = math.fsum(self.len) / self.n
ctf, cdf = [], Counter()
for i, (d, s, e) in enumerate(ch):
for t, f in Counter(docs[d][s:e]).items():
self.post[t].append((i, f))
ctf.append(Counter(concept(t) for t in docs[d][s:e]))
cdf.update(ctf[-1].keys())
self.idf = {c: math.log(self.n / v) for c, v in cdf.items()}
self.norm = []
for i, cf in enumerate(ctf):
xs = [(1 + math.log(v)) * self.idf[c] for c, v in cf.items()]
for (c, _), x in zip(cf.items(), xs):
self.cpost[c].append((i, x))
self.norm.append(math.sqrt(math.fsum(x * x for x in xs)) or 1.0)
def bm25(self, q):
sc = defaultdict(list)
for t in q:
p = self.post.get(t, [])
idf = math.log(1 + (self.n - len(p) + 0.5) / (len(p) + 0.5))
for i, f in p:
den = f + self.k1 * (1 - self.b + self.b * self.len[i]
/ self.avg)
sc[i].append(idf * f * (self.k1 + 1) / den)
return {i: math.fsum(v) for i, v in sc.items()}
def cos(self, q):
qc = Counter(concept(t) for t in q)
qw = {c: (1 + math.log(v)) * self.idf[c]
for c, v in qc.items() if c in self.idf}
qn = math.sqrt(math.fsum(x * x for x in qw.values())) or 1.0
sc = defaultdict(list)
for c, x in qw.items():
for i, y in self.cpost[c]:
sc[i].append(x * y)
return {i: math.fsum(v) / (qn * self.norm[i])
for i, v in sc.items()}
def scale(s):
# 理论最小值为零的极大值归一:分数 / 本查询最高分
hi = max(s.values(), default=0.0) or 1.0
return {i: v / hi for i, v in s.items()}
def pair(ix, q):
return scale(ix.cos(q)), scale(ix.bm25(q))
def rank(a, b, alpha, k):
# 混合分 = alpha * 向量侧 + (1 - alpha) * 词频侧
sc = [(-(alpha * a.get(i, 0.0) + (1 - alpha) * b.get(i, 0.0)), i)
for i in sorted(set(a) | set(b))]
return [i for _, i in heapq.nsmallest(k, sc)]
def hit(ix, got, d, a, b, need=0.8):
# 命中:取回的块合起来覆盖事实段至少八成
cov = set()
for i in got:
dd, s, e = ix.ch[i]
if dd == d:
cov.update(range(max(s, a), min(e, b)))
return len(cov) >= need * (b - a)
ALPHA, BUDGET = 0.7, 192
SIZES = (8, 16, 24, 32, 48, 64, 96, 192)
print("span " + " ".join("%6d" % w for w in SIZES) + " best w/span")
for span in (12, 24, 48):
docs, facts = corpus(ndoc=160, span=span)
qs = queries(facts)
row = []
for w in SIZES:
ix = Index(docs, chunks(docs, w, 0.0))
kb = max(1, BUDGET // w)
h = 0
for typ, q, d, a, b in qs:
h += hit(ix, rank(*pair(ix, q), ALPHA, kb), d, a, b)
row.append(h / len(qs))
j = row.index(max(row))
print("%4d " % span + " ".join("%6.3f" % r for r in row)
+ " %4d %6.2f" % (SIZES[j], SIZES[j] / span))唯一数据源与引用
范雅福(Fan, Y.). (2026). 《知识库分块边界网格寻优、重叠率与向量词频混合检索配比母典》第 1 章「倒 U:阅读预算下的块长与召回」(版本 1.0.0). 5D Library. DOI 待发布
DOI 待发布 · 专著页正式发布后上线(本站暂无专著页) · 5DL-B-261 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026261,
author = {Fan, Yafu},
title = {Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
note = {DOI pending}
}