What does Compression Distance and Rewrite Detection conclude, in numbers, about “Deep Paraphrase: Number Fingerprints and Argument-Sequence Fingerprints”?

Answer: Chapter 5 takes on deep paraphrase: the area under the curve of compression distance is only 0.0949, the ordered number fingerprint detects 0.923, and the argument-sequence fingerprint detects only 0.050 because of genre conventions.

Scope of the numbers: Its corpora, thresholds and parameters are illustrative, every sample text is generated by program or written by the author, and all calculations are generated by its programs. It is not individual business advice and not legal advice, and every flag raised by its small tool needs human review.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 0.050, 0.0949, 0.923

featurenull p99light detdeep detdeep AUCdeep mean
J30.00711.0000.2130.72920.0041
1-NCD0.25411.0000.0000.09490.1724
NUM0.60000.9930.9230.98770.8676
MOV0.83331.0000.0500.79150.6678
NUM+MOV0.62500.9930.8600.98760.7677
mean NUM of independent pairs: 0.2110; mean MOV: 0.5211
Show the full program
# Chapter 5. p51: deep paraphrase defeats n-grams and NCD, not fingerprints.
# Sales scripts of 12 sentences: each sentence opens with a discourse
# marker of its move (hook, pain, claim, proof, number, offer, objection,
# close), and 35 % carry a number. Independent scripts of the same genre
# follow the usual move order with chance 0.6, so they resemble each
# other in structure. A deep paraphrase rewords 90 % of content words,
# swaps half the function words, rotates each clause, drops, inserts and
# swaps sentences and loses 20 % of markers; numbers and the order of
# moves survive. Fingerprints: LCS similarity of the number sequence
# (NUM) and of the marker sequence (MOV). 300 pairs per class; detection
# at the 99th percentile of 300 same-genre independent pairs. Illustrative.
import bisect
import math
import random
import zlib

CONS, VOWS = "bdfgklmnprstvz", "aeiou"


def lexicon(seed, n_content=400, n_forms=3, n_func=30):
    rng = random.Random(seed)
    seen = set()

    def word(k):
        while True:
            w = "".join(rng.choice(CONS) + rng.choice(VOWS)
                        for _ in range(k))
            if w not in seen:
                seen.add(w)
                return w
    func = [word(1) for _ in range(n_func)]
    content = [[word(rng.randint(2, 3)) for _ in range(n_forms)]
               for _ in range(n_content)]
    return func, content


def zipf_cum(n):
    w = [1.0 / (k + 1) for k in range(n)]
    tot = math.fsum(w)
    return [math.fsum(w[:k + 1]) / tot for k in range(n)]


FUNC, CONTENT = lexicon(293)
FCUM, CCUM = zipf_cum(len(FUNC)), zipf_cum(len(CONTENT))


def draw(rng, cum):
    return min(bisect.bisect_left(cum, rng.random()), len(cum) - 1)


def script(rng, n_words, p_func=0.4):
    toks = []
    while len(toks) < n_words:
        for _ in range(rng.randint(8, 14)):
            if rng.random() < p_func:
                toks.append(("f", draw(rng, FCUM), 0))
            else:
                toks.append(("c", draw(rng, CCUM), 0))
        toks.append(("p", 0, 0))
    return toks


def render(toks):
    out = []
    for kind, i, form in toks:
        if kind == "f":
            out.append(FUNC[i])
        elif kind == "c":
            out.append(CONTENT[i][form])
        elif out:
            out[-1] += "."
    return " ".join(out)


def substitute(rng, toks, r):
    out = []
    for kind, i, form in toks:
        if kind == "c" and rng.random() < r:
            form = rng.choice([k for k in range(3) if k != form])
        out.append((kind, i, form))
    return out


def C(b):
    return len(zlib.compress(b, 9))


def ncd(bx, by):
    cx, cy, cxy = C(bx), C(by), C(bx + by)
    return (cxy - min(cx, cy)) / max(cx, cy)


MOVES = "HPCRNOXK"   # hook pain claim proof number offer objection close
NUMS = [3, 7, 9, 12, 15, 19, 24, 30, 49, 59, 99, 128, 199, 299, 399, 999]
NCUM = zipf_cum(len(NUMS))
mrng = random.Random(2930)
# two discourse markers per move, three forms each, four syllables long
MARK = [["".join(mrng.choice(CONS) + mrng.choice(VOWS) for _ in range(4))
         for _ in range(3)] for _ in range(16)]
MARKER = {MARK[j][f]: MOVES[j // 2] for j in range(16) for f in range(3)}


def sales_script(rng, n_sent=12, p_num=0.35, conv=0.6):
    # sentence = [move, marker, form, body tokens, number or None];
    # with chance conv the next move follows the usual genre order
    out, m = [], "H"
    for _ in range(n_sent):
        body = script(rng, 7)[:-1]
        num = NUMS[draw(rng, NCUM)] if rng.random() < p_num else None
        out.append([m, rng.randrange(2), 0, body, num])
        if rng.random() < conv:
            m = MOVES[min(MOVES.index(m) + 1, 7)]
        else:
            m = rng.choice(MOVES)
    return out


def say(sents):
    words = []
    for m, k, form, body, num in sents:
        if m != "?":
            words.append(MARK[2 * MOVES.index(m) + k][form])
        words.append(render(body))
        if num is not None:
            words.append(str(num))
        words[-1] += "."
    return " ".join(words)


def reword(rng, body, r, p_func):
    out = []
    for kind, i, form in body:
        if kind == "c" and rng.random() < r:
            form = rng.choice([x for x in range(3) if x != form])
        elif kind == "f" and rng.random() < p_func:
            i = rng.randrange(30)
        out.append((kind, i, form))
    return out


def light_rewrite(rng, sents, r=0.3):
    return [[m, k, f, reword(rng, b, r, 0.0), n]
            for m, k, f, b, n in sents]


def deep_rewrite(rng, sents, r=0.9):
    # reword, rotate clauses, drop 10 %, insert 15 %, lose 20 % of
    # markers, swap neighbours; the numbers and move order stay
    out = []
    for m, k, form, body, num in sents:
        if rng.random() < 0.1:
            continue
        body = reword(rng, body, r, 0.5)
        cut = rng.randrange(1, len(body))
        mk = "?" if rng.random() < 0.2 else m
        out.append([mk, k, rng.randrange(3), body[cut:] + body[:cut], num])
        if rng.random() < 0.15:
            out.append(["?", 0, 0, script(rng, 7)[:-1], None])
    for i in range(len(out) - 1):
        if rng.random() < 0.15:
            out[i], out[i + 1] = out[i + 1], out[i]
    return out


def fingerprint(text):
    nums, moves = [], []
    for sent in text.split(". "):
        w = sent.rstrip(".").split()
        moves.append(MARKER.get(w[0], "?") if w else "?")
        nums += [x for x in w if x.isdigit()]
    return nums, "".join(m for m in moves if m != "?")


def lcs(a, b):
    prev = [0] * (len(b) + 1)
    for x in a:
        cur = [0]
        for j, y in enumerate(b):
            cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
        prev = cur
    return prev[-1]


def lcs_sim(a, b):
    return lcs(a, b) / max(len(a), len(b), 1)


def feats(a, b):
    wa, wb = a.split(), b.split()
    j3 = jac({tuple(wa[i:i + 3]) for i in range(len(wa) - 2)},
             {tuple(wb[i:i + 3]) for i in range(len(wb) - 2)})
    na, ma = fingerprint(a)
    nb, mb = fingerprint(b)
    num, mov = lcs_sim(na, nb), lcs_sim(ma, mb)
    return [j3, 1.0 - ncd(a.encode(), b.encode()), num, mov,
            (num + mov) / 2.0]


def jac(x, y):
    return len(x & y) / len(x | y)


def auc(pos, neg):
    neg = sorted(neg)
    win = []
    for p in pos:
        lo, hi = bisect.bisect_left(neg, p), bisect.bisect_right(neg, p)
        win.append(lo + 0.5 * (hi - lo))
    return math.fsum(win) / (len(pos) * len(neg))


rng = random.Random(20261113)
P = 300
null, light, deep = [], [], []
for _ in range(P):
    s = sales_script(rng)
    a = say(s)
    null.append(feats(a, say(sales_script(rng))))
    light.append(feats(a, say(light_rewrite(rng, s))))
    deep.append(feats(a, say(deep_rewrite(rng, s))))
names = ["J3", "1-NCD", "NUM", "MOV", "NUM+MOV"]
print("feature   null p99  light det  deep det  deep AUC  deep mean")
for m, nm in enumerate(names):
    col = sorted(x[m] for x in null)
    bar = col[int(0.99 * P)]
    dl = len([x for x in light if x[m] > bar]) / P
    dd = len([x for x in deep if x[m] > bar]) / P
    print("%-8s  %.4f    %.3f      %.3f     %.4f    %.4f"
          % (nm, bar, dl, dd, auc([x[m] for x in deep], col),
             math.fsum(x[m] for x in deep) / P))
print("mean NUM of independent pairs: %.4f; mean MOV: %.4f"
      % (math.fsum(x[2] for x in null) / P,
         math.fsum(x[3] for x in null) / P))

Single source and citation

Fan, Y. (2026). Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts (Version 1.0.0), Chapter 5: Deep Paraphrase: Number Fingerprints and Argument-Sequence Fingerprints. 5D Library. DOI pending

DOI pending · monograph page goes live after publication · 5DL-B-293 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026293,
  author    = {Fan, Yafu},
  title     = {Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

Other chapter answers in this book