关于「重叠率:边界切断的精确概率与收益递减的拐点」,《分块与混合配比》给出的定量结论是什么?
结论:块长 32 时,召回从不重叠的 0.584 一路升到重叠一半的 0.970,与 p41 预测的饱和点吻合;重叠到四分之三,块数从 2400 涨到 8400,召回反而略降到 0.964,取回文字里重复的比例从 0.120 升到 0.365。块长 64 时,召回在重叠 0.375 处达到 0.873,之后下降,重叠一半时 0.851,四分之三时 0.824,重复比例 0.331。
数据口径:书中参数为示意;全部计算由书中程序生成;程序不读写任何文件,不连接任何线上知识库;本书是非个人化经营建议,任何线上知识库参数的改动须经离线评测并由主事者人工复核放行。 第五章扫描混合权重:编号型查询最优 0.0,改写型最优 1.0,示意基线 0.7 / 0.3 在全改写型时遗憾 0.098;按类型分流在判别出错率为 0.10 时留出召回 0.928,已低于调好的全局权重 0.938。
对照数据表
下表数字由本章 Python 程序原样打印,可重新运行复现。 程序只用 Python 标准库与结论吻合的打印值: 0.120, 0.331, 0.365, 0.375, 0.584, 0.824
| w | o | chunks | recall | dup-share |
|---|---|---|---|---|
| 32 | 0.000 | 2400 | 0.584 | 0.000 |
| 32 | 0.125 | 2640 | 0.725 | 0.009 |
| 32 | 0.250 | 3120 | 0.833 | 0.027 |
| 32 | 0.375 | 3600 | 0.926 | 0.064 |
| 32 | 0.500 | 4320 | 0.970 | 0.120 |
| 32 | 0.750 | 8400 | 0.964 | 0.365 |
| 64 | 0.000 | 1200 | 0.677 | 0.000 |
| 64 | 0.125 | 1440 | 0.764 | 0.006 |
| 64 | 0.250 | 1440 | 0.852 | 0.019 |
| 64 | 0.375 | 1680 | 0.873 | 0.052 |
| 64 | 0.500 | 2160 | 0.851 | 0.108 |
| 64 | 0.750 | 3840 | 0.824 | 0.331 |
查看完整验算程序
# Chapter 4. p42: overlap in the real pipeline: returns diminish, then
# duplicates eat the budget. Model: corpus as p11 (240 documents),
# hybrid 0.7 / 0.3, budget 192 tokens (K = 192 // w). For each overlap:
# index size, recall, and the share of returned tokens that are copies
# of tokens already returned (wasted budget). Illustrative.
import bisect
import heapq
import math
import random
from collections import Counter, defaultdict
SEED = 20261113
random.seed(SEED) # 全局种子;各处另用独立的 Random 实例
def corpus(ndoc=240, doclen=300, span=24, seed=SEED):
# 合成语料:每篇一个主题,两段事实;事实 = 一个编号 + 四个要点概念
# 每个概念有三种写法(x、y、z),模拟同义改写
rng = random.Random(seed)
w = [1.0 / i for i in range(1, 301)]
cdf = [math.fsum(w[:i + 1]) / math.fsum(w) for i in range(300)]
cdf[-1] = 1.0
docs, facts = [], []
for d in range(ndoc):
t, toks = d % 10, []
for _ in range(doclen):
if rng.random() < 0.5:
j = rng.randrange(45)
c = ("t%02d%02d" % (t, j) if j < 25
else "k%02d%02d" % (t, j - 25))
else:
c = "b%03d" % bisect.bisect_left(cdf, rng.random())
toks.append(c + "xyz"[rng.randrange(3)])
while True:
a1 = rng.randrange(doclen - span + 1)
a2 = rng.randrange(doclen - span + 1)
if abs(a1 - a2) >= span:
break
rep = 2 if span >= 9 else 1
for a in sorted((a1, a2)):
keys = rng.sample(range(20), 4)
forms = [rng.randrange(3) for _ in keys]
pos = rng.sample(range(a, a + span), 1 + 4 * rep)
toks[pos[0]] = "id%04d" % len(facts)
for j, k in enumerate(keys):
for p in pos[1 + rep * j:1 + rep * (j + 1)]:
toks[p] = "k%02d%02d%s" % (t, k, "xyz"[forms[j]])
facts.append((d, a, a + span, t, keys, forms))
docs.append(toks)
return docs, facts
def queries(facts, keep=0.25, seed=SEED + 1):
# 编号型:编号 + 两个原文写法的要点;改写型:四个要点,多数换写法
rng = random.Random(seed)
out = []
for n, (d, a, b, t, keys, forms) in enumerate(facts):
kw = ["id%04d" % n]
for j in rng.sample(range(4), 2):
kw.append("k%02d%02d%s" % (t, keys[j], "xyz"[forms[j]]))
pa = []
for j in range(4):
f = forms[j]
if rng.random() > keep:
f = (f + 1 + rng.randrange(2)) % 3
pa.append("k%02d%02d%s" % (t, keys[j], "xyz"[f]))
out.append(("id", kw, d, a, b))
out.append(("para", pa, d, a, b))
return out
def concept(tok):
# 向量侧的示意:同义写法并为一个概念,所有编号并为一个概念
return "ID" if tok.startswith("id") else tok[:-1]
def chunks(docs, w, ov):
step, out = max(1, int(round(w * (1 - ov)))), []
for d, toks in enumerate(docs):
s = 0
while True:
out.append((d, s, min(s + w, len(toks))))
if s + w >= len(toks):
break
s += step
return out
class Index:
def __init__(self, docs, ch, k1=1.2, b=0.75):
self.ch, self.k1, self.b, self.n = ch, k1, b, len(ch)
self.post, self.cpost = defaultdict(list), defaultdict(list)
self.len = [e - s for _, s, e in ch]
self.avg = math.fsum(self.len) / self.n
ctf, cdf = [], Counter()
for i, (d, s, e) in enumerate(ch):
for t, f in Counter(docs[d][s:e]).items():
self.post[t].append((i, f))
ctf.append(Counter(concept(t) for t in docs[d][s:e]))
cdf.update(ctf[-1].keys())
self.idf = {c: math.log(self.n / v) for c, v in cdf.items()}
self.norm = []
for i, cf in enumerate(ctf):
xs = [(1 + math.log(v)) * self.idf[c] for c, v in cf.items()]
for (c, _), x in zip(cf.items(), xs):
self.cpost[c].append((i, x))
self.norm.append(math.sqrt(math.fsum(x * x for x in xs)) or 1.0)
def bm25(self, q):
sc = defaultdict(list)
for t in q:
p = self.post.get(t, [])
idf = math.log(1 + (self.n - len(p) + 0.5) / (len(p) + 0.5))
for i, f in p:
den = f + self.k1 * (1 - self.b + self.b * self.len[i]
/ self.avg)
sc[i].append(idf * f * (self.k1 + 1) / den)
return {i: math.fsum(v) for i, v in sc.items()}
def cos(self, q):
qc = Counter(concept(t) for t in q)
qw = {c: (1 + math.log(v)) * self.idf[c]
for c, v in qc.items() if c in self.idf}
qn = math.sqrt(math.fsum(x * x for x in qw.values())) or 1.0
sc = defaultdict(list)
for c, x in qw.items():
for i, y in self.cpost[c]:
sc[i].append(x * y)
return {i: math.fsum(v) / (qn * self.norm[i])
for i, v in sc.items()}
def scale(s):
# 理论最小值为零的极大值归一:分数 / 本查询最高分
hi = max(s.values(), default=0.0) or 1.0
return {i: v / hi for i, v in s.items()}
def pair(ix, q):
return scale(ix.cos(q)), scale(ix.bm25(q))
def rank(a, b, alpha, k):
# 混合分 = alpha * 向量侧 + (1 - alpha) * 词频侧
sc = [(-(alpha * a.get(i, 0.0) + (1 - alpha) * b.get(i, 0.0)), i)
for i in sorted(set(a) | set(b))]
return [i for _, i in heapq.nsmallest(k, sc)]
def hit(ix, got, d, a, b, need=0.8):
# 命中:取回的块合起来覆盖事实段至少八成
cov = set()
for i in got:
dd, s, e = ix.ch[i]
if dd == d:
cov.update(range(max(s, a), min(e, b)))
return len(cov) >= need * (b - a)
docs, facts = corpus()
qs = queries(facts)
print(" w o chunks recall dup-share")
for w in (32, 64):
for o in (0.0, 0.125, 0.25, 0.375, 0.5, 0.75):
ix = Index(docs, chunks(docs, w, o))
k, h, dup, tot = max(1, 192 // w), 0, 0, 0
for typ, q, d, a, b in qs:
got = rank(*pair(ix, q), 0.7, k)
h += hit(ix, got, d, a, b)
seen = set()
for i in got:
dd, s, e = ix.ch[i]
for p in range(s, e):
dup += (dd, p) in seen
seen.add((dd, p))
tot += e - s
print("%4d %5.3f %6d %6.3f %9.3f" % (w, o, ix.n, h / len(qs),
dup / tot))唯一数据源与引用
范雅福(Fan, Y.). (2026). 《知识库分块边界网格寻优、重叠率与向量词频混合检索配比母典》第 4 章「重叠率:边界切断的精确概率与收益递减的拐点」(版本 1.0.0). 5D Library. DOI 待发布
DOI 待发布 · 专著页正式发布后上线(本站暂无专著页) · 5DL-B-261 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026261,
author = {Fan, Yafu},
title = {Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
note = {DOI pending}
}