What does Chunking and Blending conclude, in numbers, about “Bag-of-Words Cosine and Dilution in Long Chunks: Which Retriever Recognises Which Query”?

Answer: The wrong chunks are more instructive. At a chunk length of 12, the strongest wrong chunk has a mean cosine of 0.584, above the 0.573 of the gold chunk, so the margin is negative. A gold chunk of 12 tokens holds only half the fact, the other half sits in the neighbouring chunk, and among thousands of small chunks in the base some always happen to gather a few key concepts. At a chunk length of 24 the gold chunk leads by the widest margin, 0.085; from 96 onwards the margin is back near zero. The share of queries in which the gold chunk beats every wrong chunk therefore also traces an inverted U: 0.528 at a length of 12, a peak of 0.787 at 24, and back down to 0.506 at 192.

Scope of the numbers: Implementing BM25 and a concept-level bag-of-words cosine in standard-library programs on fixed-seed synthetic corpora, the book corrects the blueprint in three places. Parameters are illustrative and all numbers come from the book's programs, which read and write no file and connect to no live knowledge base.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 0.085, 0.506, 0.528, 0.573, 0.584, 0.787

wcos-goldcos-wrongmarginsqrt-lawwin-coswin-bm25
120.5730.584-0.0110.5730.5280.081
240.5910.5060.0850.4050.7870.113
480.5080.4630.0450.2870.7690.122
960.4200.426-0.0060.2030.5780.144
1920.3810.3790.0020.1430.5060.141
Show the full program
# Chapter 3. p31: a longer chunk dilutes the cosine of the right chunk.
# Model: corpus as p11 (160 documents), paraphrase queries only. For each
# chunk length w: mean cosine of the gold chunk (the one holding most of
# the fact), mean cosine of the best wrong chunk, the share of queries in
# which gold beats every wrong chunk, and the same share for BM25.
# The sqrt column is the naive law cos ~ sqrt(12 / w) scaled at w = 12.
import bisect
import heapq
import math
import random
from collections import Counter, defaultdict

SEED = 20261113
random.seed(SEED)  # 全局种子;各处另用独立的 Random 实例


def corpus(ndoc=240, doclen=300, span=24, seed=SEED):
    # 合成语料:每篇一个主题,两段事实;事实 = 一个编号 + 四个要点概念
    # 每个概念有三种写法(x、y、z),模拟同义改写
    rng = random.Random(seed)
    w = [1.0 / i for i in range(1, 301)]
    cdf = [math.fsum(w[:i + 1]) / math.fsum(w) for i in range(300)]
    cdf[-1] = 1.0
    docs, facts = [], []
    for d in range(ndoc):
        t, toks = d % 10, []
        for _ in range(doclen):
            if rng.random() < 0.5:
                j = rng.randrange(45)
                c = ("t%02d%02d" % (t, j) if j < 25
                     else "k%02d%02d" % (t, j - 25))
            else:
                c = "b%03d" % bisect.bisect_left(cdf, rng.random())
            toks.append(c + "xyz"[rng.randrange(3)])
        while True:
            a1 = rng.randrange(doclen - span + 1)
            a2 = rng.randrange(doclen - span + 1)
            if abs(a1 - a2) >= span:
                break
        rep = 2 if span >= 9 else 1
        for a in sorted((a1, a2)):
            keys = rng.sample(range(20), 4)
            forms = [rng.randrange(3) for _ in keys]
            pos = rng.sample(range(a, a + span), 1 + 4 * rep)
            toks[pos[0]] = "id%04d" % len(facts)
            for j, k in enumerate(keys):
                for p in pos[1 + rep * j:1 + rep * (j + 1)]:
                    toks[p] = "k%02d%02d%s" % (t, k, "xyz"[forms[j]])
            facts.append((d, a, a + span, t, keys, forms))
        docs.append(toks)
    return docs, facts


def queries(facts, keep=0.25, seed=SEED + 1):
    # 编号型:编号 + 两个原文写法的要点;改写型:四个要点,多数换写法
    rng = random.Random(seed)
    out = []
    for n, (d, a, b, t, keys, forms) in enumerate(facts):
        kw = ["id%04d" % n]
        for j in rng.sample(range(4), 2):
            kw.append("k%02d%02d%s" % (t, keys[j], "xyz"[forms[j]]))
        pa = []
        for j in range(4):
            f = forms[j]
            if rng.random() > keep:
                f = (f + 1 + rng.randrange(2)) % 3
            pa.append("k%02d%02d%s" % (t, keys[j], "xyz"[f]))
        out.append(("id", kw, d, a, b))
        out.append(("para", pa, d, a, b))
    return out


def concept(tok):
    # 向量侧的示意:同义写法并为一个概念,所有编号并为一个概念
    return "ID" if tok.startswith("id") else tok[:-1]


def chunks(docs, w, ov):
    step, out = max(1, int(round(w * (1 - ov)))), []
    for d, toks in enumerate(docs):
        s = 0
        while True:
            out.append((d, s, min(s + w, len(toks))))
            if s + w >= len(toks):
                break
            s += step
    return out


class Index:
    def __init__(self, docs, ch, k1=1.2, b=0.75):
        self.ch, self.k1, self.b, self.n = ch, k1, b, len(ch)
        self.post, self.cpost = defaultdict(list), defaultdict(list)
        self.len = [e - s for _, s, e in ch]
        self.avg = math.fsum(self.len) / self.n
        ctf, cdf = [], Counter()
        for i, (d, s, e) in enumerate(ch):
            for t, f in Counter(docs[d][s:e]).items():
                self.post[t].append((i, f))
            ctf.append(Counter(concept(t) for t in docs[d][s:e]))
            cdf.update(ctf[-1].keys())
        self.idf = {c: math.log(self.n / v) for c, v in cdf.items()}
        self.norm = []
        for i, cf in enumerate(ctf):
            xs = [(1 + math.log(v)) * self.idf[c] for c, v in cf.items()]
            for (c, _), x in zip(cf.items(), xs):
                self.cpost[c].append((i, x))
            self.norm.append(math.sqrt(math.fsum(x * x for x in xs)) or 1.0)

    def bm25(self, q):
        sc = defaultdict(list)
        for t in q:
            p = self.post.get(t, [])
            idf = math.log(1 + (self.n - len(p) + 0.5) / (len(p) + 0.5))
            for i, f in p:
                den = f + self.k1 * (1 - self.b + self.b * self.len[i]
                                     / self.avg)
                sc[i].append(idf * f * (self.k1 + 1) / den)
        return {i: math.fsum(v) for i, v in sc.items()}

    def cos(self, q):
        qc = Counter(concept(t) for t in q)
        qw = {c: (1 + math.log(v)) * self.idf[c]
              for c, v in qc.items() if c in self.idf}
        qn = math.sqrt(math.fsum(x * x for x in qw.values())) or 1.0
        sc = defaultdict(list)
        for c, x in qw.items():
            for i, y in self.cpost[c]:
                sc[i].append(x * y)
        return {i: math.fsum(v) / (qn * self.norm[i])
                for i, v in sc.items()}


def scale(s):
    # 理论最小值为零的极大值归一:分数 / 本查询最高分
    hi = max(s.values(), default=0.0) or 1.0
    return {i: v / hi for i, v in s.items()}


def pair(ix, q):
    return scale(ix.cos(q)), scale(ix.bm25(q))


def rank(a, b, alpha, k):
    # 混合分 = alpha * 向量侧 + (1 - alpha) * 词频侧
    sc = [(-(alpha * a.get(i, 0.0) + (1 - alpha) * b.get(i, 0.0)), i)
          for i in sorted(set(a) | set(b))]
    return [i for _, i in heapq.nsmallest(k, sc)]


def hit(ix, got, d, a, b, need=0.8):
    # 命中:取回的块合起来覆盖事实段至少八成
    cov = set()
    for i in got:
        dd, s, e = ix.ch[i]
        if dd == d:
            cov.update(range(max(s, a), min(e, b)))
    return len(cov) >= need * (b - a)


docs, facts = corpus(ndoc=160)
qs = [x for x in queries(facts) if x[0] == "para"]
print("   w  cos-gold  cos-wrong  margin  sqrt-law  win-cos  win-bm25")
base = None
for w in (12, 24, 48, 96, 192):
    ix = Index(docs, chunks(docs, w, 0.0))
    g, o, wc, wb = [], [], 0, 0
    for typ, q, d, a, b in qs:
        best, gi = -1, None
        for i, (dd, s, e) in enumerate(ix.ch):
            if dd == d and min(e, b) - max(s, a) > best:
                best, gi = min(e, b) - max(s, a), i
        for sc, tag in ((ix.cos(q), "c"), (ix.bm25(q), "b")):
            gold = sc.get(gi, 0.0)
            wrong = max((v for i, v in sc.items() if i != gi), default=0)
            if tag == "c":
                g.append(gold)
                o.append(wrong)
                wc += gold > wrong
            else:
                wb += gold > wrong
    mg = math.fsum(g) / len(g)
    base = base or mg
    print("%4d  %8.3f  %9.3f  %6.3f  %8.3f  %7.3f  %8.3f" % (
        w, mg, math.fsum(o) / len(o), mg - math.fsum(o) / len(o),
        base * math.sqrt(12 / w), wc / len(qs), wb / len(qs)))

Single source and citation

Fan, Y. (2026). Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval (Version 1.0.0), Chapter 3: Bag-of-Words Cosine and Dilution in Long Chunks: Which Retriever Recognises Which Query. 5D Library. DOI pending

DOI pending · monograph page goes live after publication · 5DL-B-261 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026261,
  author    = {Fan, Yafu},
  title     = {Grid Search over Knowledge-Base Chunk Boundaries: Overlap Ratios and the Hybrid Weighting of Vector and Term-Frequency Retrieval},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

Other chapter answers in this book