What does Compression Distance and Rewrite Detection conclude, in numbers, about “Deep Paraphrase: Number Fingerprints and Argument-Sequence Fingerprints”?
Answer: Chapter 5 takes on deep paraphrase: the area under the curve of compression distance is only 0.0949, the ordered number fingerprint detects 0.923, and the argument-sequence fingerprint detects only 0.050 because of genre conventions.
Scope of the numbers: Its corpora, thresholds and parameters are illustrative, every sample text is generated by program or written by the author, and all calculations are generated by its programs. It is not individual business advice and not legal advice, and every flag raised by its small tool needs human review.
Data table
The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 0.050, 0.0949, 0.923
| feature | null p99 | light det | deep det | deep AUC | deep mean |
| J3 | 0.0071 | 1.000 | 0.213 | 0.7292 | 0.0041 |
| 1-NCD | 0.2541 | 1.000 | 0.000 | 0.0949 | 0.1724 |
| NUM | 0.6000 | 0.993 | 0.923 | 0.9877 | 0.8676 |
| MOV | 0.8333 | 1.000 | 0.050 | 0.7915 | 0.6678 |
| NUM+MOV | 0.6250 | 0.993 | 0.860 | 0.9876 | 0.7677 |
| mean NUM of independent pairs: 0.2110; mean MOV: 0.5211 |
Show the full program
# Chapter 5. p51: deep paraphrase defeats n-grams and NCD, not fingerprints.
# Sales scripts of 12 sentences: each sentence opens with a discourse
# marker of its move (hook, pain, claim, proof, number, offer, objection,
# close), and 35 % carry a number. Independent scripts of the same genre
# follow the usual move order with chance 0.6, so they resemble each
# other in structure. A deep paraphrase rewords 90 % of content words,
# swaps half the function words, rotates each clause, drops, inserts and
# swaps sentences and loses 20 % of markers; numbers and the order of
# moves survive. Fingerprints: LCS similarity of the number sequence
# (NUM) and of the marker sequence (MOV). 300 pairs per class; detection
# at the 99th percentile of 300 same-genre independent pairs. Illustrative.
import bisect
import math
import random
import zlib
CONS, VOWS = "bdfgklmnprstvz", "aeiou"
def lexicon(seed, n_content=400, n_forms=3, n_func=30):
rng = random.Random(seed)
seen = set()
def word(k):
while True:
w = "".join(rng.choice(CONS) + rng.choice(VOWS)
for _ in range(k))
if w not in seen:
seen.add(w)
return w
func = [word(1) for _ in range(n_func)]
content = [[word(rng.randint(2, 3)) for _ in range(n_forms)]
for _ in range(n_content)]
return func, content
def zipf_cum(n):
w = [1.0 / (k + 1) for k in range(n)]
tot = math.fsum(w)
return [math.fsum(w[:k + 1]) / tot for k in range(n)]
FUNC, CONTENT = lexicon(293)
FCUM, CCUM = zipf_cum(len(FUNC)), zipf_cum(len(CONTENT))
def draw(rng, cum):
return min(bisect.bisect_left(cum, rng.random()), len(cum) - 1)
def script(rng, n_words, p_func=0.4):
toks = []
while len(toks) < n_words:
for _ in range(rng.randint(8, 14)):
if rng.random() < p_func:
toks.append(("f", draw(rng, FCUM), 0))
else:
toks.append(("c", draw(rng, CCUM), 0))
toks.append(("p", 0, 0))
return toks
def render(toks):
out = []
for kind, i, form in toks:
if kind == "f":
out.append(FUNC[i])
elif kind == "c":
out.append(CONTENT[i][form])
elif out:
out[-1] += "."
return " ".join(out)
def substitute(rng, toks, r):
out = []
for kind, i, form in toks:
if kind == "c" and rng.random() < r:
form = rng.choice([k for k in range(3) if k != form])
out.append((kind, i, form))
return out
def C(b):
return len(zlib.compress(b, 9))
def ncd(bx, by):
cx, cy, cxy = C(bx), C(by), C(bx + by)
return (cxy - min(cx, cy)) / max(cx, cy)
MOVES = "HPCRNOXK" # hook pain claim proof number offer objection close
NUMS = [3, 7, 9, 12, 15, 19, 24, 30, 49, 59, 99, 128, 199, 299, 399, 999]
NCUM = zipf_cum(len(NUMS))
mrng = random.Random(2930)
# two discourse markers per move, three forms each, four syllables long
MARK = [["".join(mrng.choice(CONS) + mrng.choice(VOWS) for _ in range(4))
for _ in range(3)] for _ in range(16)]
MARKER = {MARK[j][f]: MOVES[j // 2] for j in range(16) for f in range(3)}
def sales_script(rng, n_sent=12, p_num=0.35, conv=0.6):
# sentence = [move, marker, form, body tokens, number or None];
# with chance conv the next move follows the usual genre order
out, m = [], "H"
for _ in range(n_sent):
body = script(rng, 7)[:-1]
num = NUMS[draw(rng, NCUM)] if rng.random() < p_num else None
out.append([m, rng.randrange(2), 0, body, num])
if rng.random() < conv:
m = MOVES[min(MOVES.index(m) + 1, 7)]
else:
m = rng.choice(MOVES)
return out
def say(sents):
words = []
for m, k, form, body, num in sents:
if m != "?":
words.append(MARK[2 * MOVES.index(m) + k][form])
words.append(render(body))
if num is not None:
words.append(str(num))
words[-1] += "."
return " ".join(words)
def reword(rng, body, r, p_func):
out = []
for kind, i, form in body:
if kind == "c" and rng.random() < r:
form = rng.choice([x for x in range(3) if x != form])
elif kind == "f" and rng.random() < p_func:
i = rng.randrange(30)
out.append((kind, i, form))
return out
def light_rewrite(rng, sents, r=0.3):
return [[m, k, f, reword(rng, b, r, 0.0), n]
for m, k, f, b, n in sents]
def deep_rewrite(rng, sents, r=0.9):
# reword, rotate clauses, drop 10 %, insert 15 %, lose 20 % of
# markers, swap neighbours; the numbers and move order stay
out = []
for m, k, form, body, num in sents:
if rng.random() < 0.1:
continue
body = reword(rng, body, r, 0.5)
cut = rng.randrange(1, len(body))
mk = "?" if rng.random() < 0.2 else m
out.append([mk, k, rng.randrange(3), body[cut:] + body[:cut], num])
if rng.random() < 0.15:
out.append(["?", 0, 0, script(rng, 7)[:-1], None])
for i in range(len(out) - 1):
if rng.random() < 0.15:
out[i], out[i + 1] = out[i + 1], out[i]
return out
def fingerprint(text):
nums, moves = [], []
for sent in text.split(". "):
w = sent.rstrip(".").split()
moves.append(MARKER.get(w[0], "?") if w else "?")
nums += [x for x in w if x.isdigit()]
return nums, "".join(m for m in moves if m != "?")
def lcs(a, b):
prev = [0] * (len(b) + 1)
for x in a:
cur = [0]
for j, y in enumerate(b):
cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
prev = cur
return prev[-1]
def lcs_sim(a, b):
return lcs(a, b) / max(len(a), len(b), 1)
def feats(a, b):
wa, wb = a.split(), b.split()
j3 = jac({tuple(wa[i:i + 3]) for i in range(len(wa) - 2)},
{tuple(wb[i:i + 3]) for i in range(len(wb) - 2)})
na, ma = fingerprint(a)
nb, mb = fingerprint(b)
num, mov = lcs_sim(na, nb), lcs_sim(ma, mb)
return [j3, 1.0 - ncd(a.encode(), b.encode()), num, mov,
(num + mov) / 2.0]
def jac(x, y):
return len(x & y) / len(x | y)
def auc(pos, neg):
neg = sorted(neg)
win = []
for p in pos:
lo, hi = bisect.bisect_left(neg, p), bisect.bisect_right(neg, p)
win.append(lo + 0.5 * (hi - lo))
return math.fsum(win) / (len(pos) * len(neg))
rng = random.Random(20261113)
P = 300
null, light, deep = [], [], []
for _ in range(P):
s = sales_script(rng)
a = say(s)
null.append(feats(a, say(sales_script(rng))))
light.append(feats(a, say(light_rewrite(rng, s))))
deep.append(feats(a, say(deep_rewrite(rng, s))))
names = ["J3", "1-NCD", "NUM", "MOV", "NUM+MOV"]
print("feature null p99 light det deep det deep AUC deep mean")
for m, nm in enumerate(names):
col = sorted(x[m] for x in null)
bar = col[int(0.99 * P)]
dl = len([x for x in light if x[m] > bar]) / P
dd = len([x for x in deep if x[m] > bar]) / P
print("%-8s %.4f %.3f %.3f %.4f %.4f"
% (nm, bar, dl, dd, auc([x[m] for x in deep], col),
math.fsum(x[m] for x in deep) / P))
print("mean NUM of independent pairs: %.4f; mean MOV: %.4f"
% (math.fsum(x[2] for x in null) / P,
math.fsum(x[3] for x in null) / P))Single source and citation
Fan, Y. (2026). Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts (Version 1.0.0), Chapter 5: Deep Paraphrase: Number Fingerprints and Argument-Sequence Fingerprints. 5D Library. DOI pending
DOI pending · monograph page goes live after publication · 5DL-B-293 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026293,
author = {Fan, Yafu},
title = {Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
note = {DOI pending}
}Other chapter answers in this book
- Chapter 1 · Sweeping the Substitution Rate: Where Literal Matching Fails
- Chapter 2 · Kolmogorov Complexity and Real Compressors: Upper Bounds, Normality and the Window
- Chapter 3 · Short-Text Bias and Threshold Sensitivity: One Line Cannot Rule Every Length
- Chapter 4 · N-grams, MinHash and Locality-Sensitive Hashing: a First Screen over Millions of Pages
- Chapter 6 · Hierarchical Clustering of Shared-Origin Script Families
- Chapter 7 · The Classics Examined: the Colophon of the Cheng House at Meishan and the Statute of Anne
- Chapter 8 · Closing the Loop: Weekly Monitoring for Rewritten Copies, De-duplicating Direct-Answer Pages and a Pre-registered Test