What does Compression Distance and Rewrite Detection conclude, in numbers, about “Closing the Loop: Weekly Monitoring for Rewritten Copies, De-duplicating Direct-Answer Pages and a Pre-registered Test”?
Answer: Chapter 8 gives 5D Library's weekly monitoring, the de-duplication of direct-answer pages, the Rewrite and Shared-Origin Detector and a pre-registered test plan: library-wide calibration of the number fingerprint cuts false alarms from 96 to 9 at the price of catching only 12 deep paraphrases, and test one needs 155 rewrites. Marked boxes separate published evidence, runnable Python calculations (21 programs), classical sources and the author's hypotheses.
Scope of the numbers: Its corpora, thresholds and parameters are illustrative, every sample text is generated by program or written by the author, and all calculations are generated by its programs. It is not individual business advice and not legal advice, and every flag raised by its small tool needs human review.
Data table
The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 12, 96
| thresholds: NCD < 0.7539, NUM pair > 0.6000, NUM bits > 13.88 | ||||||
| type | pages | LSH | NCD | NUM pair | NUM bits | to review |
| near copy | 10 | 10 | 10 | 10 | 4 | 10 |
| light | 20 | 20 | 20 | 20 | 10 | 20 |
| deep | 30 | 0 | 0 | 28 | 12 | 12 |
| independent | 340 | 1 | 0 | 96 | 9 | 9 |
Show the full program
# Chapter 8. p81: a weekly monitoring run, end to end, on a toy corpus.
# Our library: 40 own pages (sales scripts as in p51). The crawl: 400
# outside pages = 340 independent same-genre pages, 10 near copies (2 %
# character errors), 20 light rewrites (30 % reworded), 30 deep
# paraphrases (as in p51). Stage 1: MinHash of word 3-grams, 128 values,
# LSH with 64 bands of 2 rows (p42). Stage 2: NCD below the 1st
# percentile of a separate null of 300 independent pairs. Stage 3: the
# number fingerprint on every pair: (a) LCS share above its pairwise
# null 99th percentile; (b) library-wise and in bits: the matched
# numbers weighted by surprisal -log2 p (p from 200 extra independent
# pages), above the 99th percentile of the best match of an independent
# page against all 40 own pages (40 comparisons per page). Every flag
# goes to a person; nothing is published or reported by the script.
# Counts are per outside page. Illustrative.
import bisect
import math
import random
import zlib
CONS, VOWS = "bdfgklmnprstvz", "aeiou"
def lexicon(seed, n_content=400, n_forms=3, n_func=30):
rng = random.Random(seed)
seen = set()
def word(k):
while True:
w = "".join(rng.choice(CONS) + rng.choice(VOWS)
for _ in range(k))
if w not in seen:
seen.add(w)
return w
func = [word(1) for _ in range(n_func)]
content = [[word(rng.randint(2, 3)) for _ in range(n_forms)]
for _ in range(n_content)]
return func, content
def zipf_cum(n):
w = [1.0 / (k + 1) for k in range(n)]
tot = math.fsum(w)
return [math.fsum(w[:k + 1]) / tot for k in range(n)]
FUNC, CONTENT = lexicon(293)
FCUM, CCUM = zipf_cum(len(FUNC)), zipf_cum(len(CONTENT))
def draw(rng, cum):
return min(bisect.bisect_left(cum, rng.random()), len(cum) - 1)
def script(rng, n_words, p_func=0.4):
toks = []
while len(toks) < n_words:
for _ in range(rng.randint(8, 14)):
if rng.random() < p_func:
toks.append(("f", draw(rng, FCUM), 0))
else:
toks.append(("c", draw(rng, CCUM), 0))
toks.append(("p", 0, 0))
return toks
def render(toks):
out = []
for kind, i, form in toks:
if kind == "f":
out.append(FUNC[i])
elif kind == "c":
out.append(CONTENT[i][form])
elif out:
out[-1] += "."
return " ".join(out)
def substitute(rng, toks, r):
out = []
for kind, i, form in toks:
if kind == "c" and rng.random() < r:
form = rng.choice([k for k in range(3) if k != form])
out.append((kind, i, form))
return out
def C(b):
return len(zlib.compress(b, 9))
def ncd(bx, by):
cx, cy, cxy = C(bx), C(by), C(bx + by)
return (cxy - min(cx, cy)) / max(cx, cy)
MOVES = "HPCRNOXK" # hook pain claim proof number offer objection close
NUMS = [3, 7, 9, 12, 15, 19, 24, 30, 49, 59, 99, 128, 199, 299, 399, 999]
NCUM = zipf_cum(len(NUMS))
mrng = random.Random(2930)
# two discourse markers per move, three forms each, four syllables long
MARK = [["".join(mrng.choice(CONS) + mrng.choice(VOWS) for _ in range(4))
for _ in range(3)] for _ in range(16)]
MARKER = {MARK[j][f]: MOVES[j // 2] for j in range(16) for f in range(3)}
def sales_script(rng, n_sent=12, p_num=0.35, conv=0.6):
# sentence = [move, marker, form, body tokens, number or None];
# with chance conv the next move follows the usual genre order
out, m = [], "H"
for _ in range(n_sent):
body = script(rng, 7)[:-1]
num = NUMS[draw(rng, NCUM)] if rng.random() < p_num else None
out.append([m, rng.randrange(2), 0, body, num])
if rng.random() < conv:
m = MOVES[min(MOVES.index(m) + 1, 7)]
else:
m = rng.choice(MOVES)
return out
def say(sents):
words = []
for m, k, form, body, num in sents:
if m != "?":
words.append(MARK[2 * MOVES.index(m) + k][form])
words.append(render(body))
if num is not None:
words.append(str(num))
words[-1] += "."
return " ".join(words)
def reword(rng, body, r, p_func):
out = []
for kind, i, form in body:
if kind == "c" and rng.random() < r:
form = rng.choice([x for x in range(3) if x != form])
elif kind == "f" and rng.random() < p_func:
i = rng.randrange(30)
out.append((kind, i, form))
return out
def light_rewrite(rng, sents, r=0.3):
return [[m, k, f, reword(rng, b, r, 0.0), n]
for m, k, f, b, n in sents]
def deep_rewrite(rng, sents, r=0.9):
# reword, rotate clauses, drop 10 %, insert 15 %, lose 20 % of
# markers, swap neighbours; the numbers and move order stay
out = []
for m, k, form, body, num in sents:
if rng.random() < 0.1:
continue
body = reword(rng, body, r, 0.5)
cut = rng.randrange(1, len(body))
mk = "?" if rng.random() < 0.2 else m
out.append([mk, k, rng.randrange(3), body[cut:] + body[:cut], num])
if rng.random() < 0.15:
out.append(["?", 0, 0, script(rng, 7)[:-1], None])
for i in range(len(out) - 1):
if rng.random() < 0.15:
out[i], out[i + 1] = out[i + 1], out[i]
return out
def fingerprint(text):
nums, moves = [], []
for sent in text.split(". "):
w = sent.rstrip(".").split()
moves.append(MARKER.get(w[0], "?") if w else "?")
nums += [x for x in w if x.isdigit()]
return nums, "".join(m for m in moves if m != "?")
def lcs(a, b):
prev = [0] * (len(b) + 1)
for x in a:
cur = [0]
for j, y in enumerate(b):
cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
prev = cur
return prev[-1]
def lcs_sim(a, b):
return lcs(a, b) / max(len(a), len(b), 1)
PR = (1 << 61) - 1
hr = random.Random(97)
AB = [(hr.randrange(1, PR), hr.randrange(PR)) for _ in range(128)]
def sig(text):
w = text.split()
g = {zlib.crc32(" ".join(w[i:i + 3]).encode())
for i in range(len(w) - 2)}
return [min((a * v + b) % PR for v in g) for a, b in AB]
def lcs_bits(a, b):
# weighted LCS: total surprisal of numbers matched in order
prev = [0.0] * (len(b) + 1)
for x in a:
cur = [0.0]
for j, y in enumerate(b):
cur.append(prev[j] + BITS.get(x, 12.0) if x == y
else max(prev[j + 1], cur[j]))
prev = cur
return prev[-1]
def typo(rng, s, e):
return "".join(rng.choice("abdefgiklmnoprstuvz") if
c.isalpha() and rng.random() < e else c for c in s)
rng = random.Random(20261113)
null_n, null_f = [], []
for _ in range(300):
a, b = say(sales_script(rng)), say(sales_script(rng))
null_n.append(ncd(a.encode(), b.encode()))
null_f.append(lcs_sim(fingerprint(a)[0], fingerprint(b)[0]))
null_n.sort()
null_f.sort()
t_ncd, t_num = null_n[3], null_f[296]
own_s = [sales_script(rng) for _ in range(40)]
own = [say(s) for s in own_s]
osf = [fingerprint(t)[0] for t in own]
extra = [fingerprint(say(sales_script(rng)))[0] for _ in range(200)]
cnt = {}
for fp in extra:
for x in fp:
cnt[x] = cnt.get(x, 0) + 1
tot = math.fsum(cnt.values())
BITS = {x: -math.log2(c / tot) for x, c in cnt.items()}
best = sorted(max(lcs_bits(f, fp) for f in osf) for fp in extra)
t_lib = best[197]
crawl = [("independent", -1, say(sales_script(rng))) for _ in range(340)]
for i in range(10):
crawl.append(("near copy", i, typo(rng, own[i], 0.02)))
for i in range(20):
crawl.append(("light", 10 + i, say(light_rewrite(rng, own_s[10 + i]))))
for i in range(30):
j = i % 40
crawl.append(("deep", j, say(deep_rewrite(rng, own_s[j]))))
osig = [sig(t) for t in own]
res = {}
for kind, src, text in crawl:
s = sig(text)
bands = [i for i in range(40)
if any(s[b:b + 2] == osig[i][b:b + 2]
for b in range(0, 128, 2))]
st2 = [i for i in bands
if ncd(own[i].encode(), text.encode()) < t_ncd]
fp = fingerprint(text)[0]
sc = [lcs_sim(osf[i], fp) for i in range(40)]
st3 = [x for x in sc if x > t_num]
st4 = [i for i in range(40) if lcs_bits(osf[i], fp) > t_lib]
r = res.setdefault(kind, [0, 0, 0, 0, 0, 0])
r[0] += 1 # step
r[1] += bool(bands) # step
r[2] += bool(st2) # step
r[3] += bool(st3) # step
r[4] += bool(st4) # step
r[5] += bool(st2 or st4) # step
print("thresholds: NCD < %.4f, NUM pair > %.4f, NUM bits > %.2f"
% (t_ncd, t_num, t_lib))
print("type pages LSH NCD NUM pair NUM bits to review")
for kind in ("near copy", "light", "deep", "independent"):
r = res[kind]
print("%-12s %4d %4d %4d %4d %4d %4d"
% ((kind,) + tuple(r)))Single source and citation
Fan, Y. (2026). Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts (Version 1.0.0), Chapter 8: Closing the Loop: Weekly Monitoring for Rewritten Copies, De-duplicating Direct-Answer Pages and a Pre-registered Test. 5D Library. DOI pending
DOI pending · monograph page goes live after publication · 5DL-B-293 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026293,
author = {Fan, Yafu},
title = {Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
note = {DOI pending}
}Other chapter answers in this book
- Chapter 1 · Sweeping the Substitution Rate: Where Literal Matching Fails
- Chapter 2 · Kolmogorov Complexity and Real Compressors: Upper Bounds, Normality and the Window
- Chapter 3 · Short-Text Bias and Threshold Sensitivity: One Line Cannot Rule Every Length
- Chapter 4 · N-grams, MinHash and Locality-Sensitive Hashing: a First Screen over Millions of Pages
- Chapter 5 · Deep Paraphrase: Number Fingerprints and Argument-Sequence Fingerprints
- Chapter 6 · Hierarchical Clustering of Shared-Origin Script Families
- Chapter 7 · The Classics Examined: the Colophon of the Cheng House at Meishan and the Statute of Anne