What does Chunking and Blending conclude, in numbers, about “The Blend: How the Best Weight Moves with the Query Mix”?
Answer: Chapter 5 sweeps the blend weight: the best value is 0.0 for exact-ID queries and 1.0 for paraphrase queries, and the illustrative 0.7 / 0.3 baseline regrets 0.098 on an all-paraphrase workload; routing by type at a classification error rate of 0.10 gives held-out recall of 0.928, already below the 0.938 of a tuned global weight.
Scope of the numbers: Implementing BM25 and a concept-level bag-of-words cosine in standard-library programs on fixed-seed synthetic corpora, the book corrects the blueprint in three places. Parameters are illustrative and all numbers come from the book's programs, which read and write no file and connect to no live knowledge base.
Data table
The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 0.0, 0.10, 0.7, 0.928, 0.938, 1.0
tuned on even facts: global 0.8, id 0.0, para 1.0 held-out recall, ID share 0.5 baseline 0.7 0.915 global tuned 0.938 routed, error 0.00 0.977 routed, error 0.05 0.952 routed, error 0.10 0.928 routed, error 0.20 0.878
Show the full program
# Chapter 5. p53: route by query type instead of one global alpha.
# Model: p51 setting. A regular expression marks a query as exact-ID when
# it holds a token like id0042. alphas are tuned on even-numbered facts
# and scored on odd-numbered facts (held out). Workload: ID share 0.5.
# A router that errs with rate e sends that share to the wrong alpha.
import bisect
import heapq
import math
import random
from collections import Counter, defaultdict
SEED = 20261113
random.seed(SEED) # 全局种子;各处另用独立的 Random 实例
def corpus(ndoc=240, doclen=300, span=24, seed=SEED):
# 合成语料:每篇一个主题,两段事实;事实 = 一个编号 + 四个要点概念
# 每个概念有三种写法(x、y、z),模拟同义改写
rng = random.Random(seed)
w = [1.0 / i for i in range(1, 301)]
cdf = [math.fsum(w[:i + 1]) / math.fsum(w) for i in range(300)]
cdf[-1] = 1.0
docs, facts = [], []
for d in range(ndoc):
t, toks = d % 10, []
for _ in range(doclen):
if rng.random() < 0.5:
j = rng.randrange(45)
c = ("t%02d%02d" % (t, j) if j < 25
else "k%02d%02d" % (t, j - 25))
else:
c = "b%03d" % bisect.bisect_left(cdf, rng.random())
toks.append(c + "xyz"[rng.randrange(3)])
while True:
a1 = rng.randrange(doclen - span + 1)
a2 = rng.randrange(doclen - span + 1)
if abs(a1 - a2) >= span:
break
rep = 2 if span >= 9 else 1
for a in sorted((a1, a2)):
keys = rng.sample(range(20), 4)
forms = [rng.randrange(3) for _ in keys]
pos = rng.sample(range(a, a + span), 1 + 4 * rep)
toks[pos[0]] = "id%04d" % len(facts)
for j, k in enumerate(keys):
for p in pos[1 + rep * j:1 + rep * (j + 1)]:
toks[p] = "k%02d%02d%s" % (t, k, "xyz"[forms[j]])
facts.append((d, a, a + span, t, keys, forms))
docs.append(toks)
return docs, facts
def queries(facts, keep=0.25, seed=SEED + 1):
# 编号型:编号 + 两个原文写法的要点;改写型:四个要点,多数换写法
rng = random.Random(seed)
out = []
for n, (d, a, b, t, keys, forms) in enumerate(facts):
kw = ["id%04d" % n]
for j in rng.sample(range(4), 2):
kw.append("k%02d%02d%s" % (t, keys[j], "xyz"[forms[j]]))
pa = []
for j in range(4):
f = forms[j]
if rng.random() > keep:
f = (f + 1 + rng.randrange(2)) % 3
pa.append("k%02d%02d%s" % (t, keys[j], "xyz"[f]))
out.append(("id", kw, d, a, b))
out.append(("para", pa, d, a, b))
return out
def concept(tok):
# 向量侧的示意:同义写法并为一个概念,所有编号并为一个概念
return "ID" if tok.startswith("id") else tok[:-1]
def chunks(docs, w, ov):
step, out = max(1, int(round(w * (1 - ov)))), []
for d, toks in enumerate(docs):
s = 0
while True:
out.append((d, s, min(s + w, len(toks))))
if s + w >= len(toks):
break
s += step
return out
class Index:
def __init__(self, docs, ch, k1=1.2, b=0.75):
self.ch, self.k1, self.b, self.n = ch, k1, b, len(ch)
self.post, self.cpost = defaultdict(list), defaultdict(list)
self.len = [e - s for _, s, e in ch]
self.avg = math.fsum(self.len) / self.n
ctf, cdf = [], Counter()
for i, (d, s, e) in enumerate(ch):
for t, f in Counter(docs[d][s:e]).items():
self.post[t].append((i, f))
ctf.append(Counter(concept(t) for t in docs[d][s:e]))
cdf.update(ctf[-1].keys())
self.idf = {c: math.log(self.n / v) for c, v in cdf.items()}
self.norm = []
for i, cf in enumerate(ctf):
xs = [(1 + math.log(v)) * self.idf[c] for c, v in cf.items()]
for (c, _), x in zip(cf.items(), xs):
self.cpost[c].append((i, x))
self.norm.append(math.sqrt(math.fsum(x * x for x in xs)) or 1.0)
def bm25(self, q):
sc = defaultdict(list)
for t in q:
p = self.post.get(t, [])
idf = math.log(1 + (self.n - len(p) + 0.5) / (len(p) + 0.5))
for i, f in p:
den = f + self.k1 * (1 - self.b + self.b * self.len[i]
/ self.avg)
sc[i].append(idf * f * (self.k1 + 1) / den)
return {i: math.fsum(v) for i, v in sc.items()}
def cos(self, q):
qc = Counter(concept(t) for t in q)
qw = {c: (1 + math.log(v)) * self.idf[c]
for c, v in qc.items() if c in self.idf}
qn = math.sqrt(math.fsum(x * x for x in qw.values())) or 1.0
sc = defaultdict(list)
for c, x in qw.items():
for i, y in self.cpost[c]:
sc[i].append(x * y)
return {i: math.fsum(v) / (qn * self.norm[i])
for i, v in sc.items()}
def scale(s):
# 理论最小值为零的极大值归一:分数 / 本查询最高分
hi = max(s.values(), default=0.0) or 1.0
return {i: v / hi for i, v in s.items()}
def pair(ix, q):
return scale(ix.cos(q)), scale(ix.bm25(q))
def rank(a, b, alpha, k):
# 混合分 = alpha * 向量侧 + (1 - alpha) * 词频侧
sc = [(-(alpha * a.get(i, 0.0) + (1 - alpha) * b.get(i, 0.0)), i)
for i in sorted(set(a) | set(b))]
return [i for _, i in heapq.nsmallest(k, sc)]
def hit(ix, got, d, a, b, need=0.8):
# 命中:取回的块合起来覆盖事实段至少八成
cov = set()
for i in got:
dd, s, e = ix.ch[i]
if dd == d:
cov.update(range(max(s, a), min(e, b)))
return len(cov) >= need * (b - a)
import re
CODE = re.compile(r"^id\d{4}$")
docs, facts = corpus()
qs = queries(facts)
ix = Index(docs, chunks(docs, 48, 0.25))
AL = [j / 10 for j in range(11)]
h = {0: defaultdict(int), 1: defaultdict(int)}
for n, (typ, q, d, a, b) in enumerate(qs):
kind = "id" if any(CODE.match(t) for t in q) else "para"
sa, sb = pair(ix, q)
for al in AL:
h[n // 2 % 2][kind, al] += hit(ix, rank(sa, sb, al, 4), d, a, b)
m = len(qs) // 4
def r(part, kind, al):
return h[part][kind, al] / m
fixed = max(AL, key=lambda x: (r(0, "id", x) + r(0, "para", x), -x))
a_id = max(AL, key=lambda x: (r(0, "id", x), -x))
a_pa = max(AL, key=lambda x: (r(0, "para", x), -x))
print("tuned on even facts: global %.1f, id %.1f, para %.1f" % (
fixed, a_id, a_pa))
print("held-out recall, ID share 0.5")
print(" baseline 0.7 %.3f" % ((r(1, "id", 0.7)
+ r(1, "para", 0.7)) / 2))
print(" global tuned %.3f" % ((r(1, "id", fixed)
+ r(1, "para", fixed)) / 2))
for e in (0.0, 0.05, 0.1, 0.2):
ri = (1 - e) * r(1, "id", a_id) + e * r(1, "id", a_pa)
rp = (1 - e) * r(1, "para", a_pa) + e * r(1, "para", a_id)
print(" routed, error %.2f %.3f" % (e, (ri + rp) / 2))Single source and citation
Fan, Y. (2026). Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval (Version 1.0.0), Chapter 5: The Blend: How the Best Weight Moves with the Query Mix. 5D Library. DOI pending
DOI pending · monograph page goes live after publication · 5DL-B-261 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026261,
author = {Fan, Yafu},
title = {Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
note = {DOI pending}
}Other chapter answers in this book
- Chapter 1 · The Inverted U: Chunk Length and Recall under a Reading Budget
- Chapter 2 · BM25 from Scratch: Inverse Document Frequency, Saturation and Length Normalisation
- Chapter 3 · Bag-of-Words Cosine and Dilution in Long Chunks: Which Retriever Recognises Which Query
- Chapter 4 · Overlap: Exact Odds of a Boundary Cut and the Knee of Diminishing Returns
- Chapter 6 · The Joint Grid and the Winner's Curse: Why the Held-Out Test Cannot Be Skipped
- Chapter 7 · Cook Ding and the Phaedrus: Two Ancient Arguments for Cutting at the Natural Joints
- Chapter 8 · The Quarterly Tuning Loop and a Plan for Refutation