What does Compression Distance and Rewrite Detection conclude, in numbers, about “Closing the Loop: Weekly Monitoring for Rewritten Copies, De-duplicating Direct-Answer Pages and a Pre-registered Test”?

Answer: Chapter 8 gives 5D Library's weekly monitoring, the de-duplication of direct-answer pages, the Rewrite and Shared-Origin Detector and a pre-registered test plan: library-wide calibration of the number fingerprint cuts false alarms from 96 to 9 at the price of catching only 12 deep paraphrases, and test one needs 155 rewrites. Marked boxes separate published evidence, runnable Python calculations (21 programs), classical sources and the author's hypotheses.

Scope of the numbers: Its corpora, thresholds and parameters are illustrative, every sample text is generated by program or written by the author, and all calculations are generated by its programs. It is not individual business advice and not legal advice, and every flag raised by its small tool needs human review.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 12, 96

thresholds: NCD < 0.7539, NUM pair > 0.6000, NUM bits > 13.88
typepagesLSHNCDNUM pairNUM bitsto review
near copy10101010410
light202020201020
deep3000281212
independent340109699
Show the full program
# Chapter 8. p81: a weekly monitoring run, end to end, on a toy corpus.
# Our library: 40 own pages (sales scripts as in p51). The crawl: 400
# outside pages = 340 independent same-genre pages, 10 near copies (2 %
# character errors), 20 light rewrites (30 % reworded), 30 deep
# paraphrases (as in p51). Stage 1: MinHash of word 3-grams, 128 values,
# LSH with 64 bands of 2 rows (p42). Stage 2: NCD below the 1st
# percentile of a separate null of 300 independent pairs. Stage 3: the
# number fingerprint on every pair: (a) LCS share above its pairwise
# null 99th percentile; (b) library-wise and in bits: the matched
# numbers weighted by surprisal -log2 p (p from 200 extra independent
# pages), above the 99th percentile of the best match of an independent
# page against all 40 own pages (40 comparisons per page). Every flag
# goes to a person; nothing is published or reported by the script.
# Counts are per outside page. Illustrative.
import bisect
import math
import random
import zlib

CONS, VOWS = "bdfgklmnprstvz", "aeiou"


def lexicon(seed, n_content=400, n_forms=3, n_func=30):
    rng = random.Random(seed)
    seen = set()

    def word(k):
        while True:
            w = "".join(rng.choice(CONS) + rng.choice(VOWS)
                        for _ in range(k))
            if w not in seen:
                seen.add(w)
                return w
    func = [word(1) for _ in range(n_func)]
    content = [[word(rng.randint(2, 3)) for _ in range(n_forms)]
               for _ in range(n_content)]
    return func, content


def zipf_cum(n):
    w = [1.0 / (k + 1) for k in range(n)]
    tot = math.fsum(w)
    return [math.fsum(w[:k + 1]) / tot for k in range(n)]


FUNC, CONTENT = lexicon(293)
FCUM, CCUM = zipf_cum(len(FUNC)), zipf_cum(len(CONTENT))


def draw(rng, cum):
    return min(bisect.bisect_left(cum, rng.random()), len(cum) - 1)


def script(rng, n_words, p_func=0.4):
    toks = []
    while len(toks) < n_words:
        for _ in range(rng.randint(8, 14)):
            if rng.random() < p_func:
                toks.append(("f", draw(rng, FCUM), 0))
            else:
                toks.append(("c", draw(rng, CCUM), 0))
        toks.append(("p", 0, 0))
    return toks


def render(toks):
    out = []
    for kind, i, form in toks:
        if kind == "f":
            out.append(FUNC[i])
        elif kind == "c":
            out.append(CONTENT[i][form])
        elif out:
            out[-1] += "."
    return " ".join(out)


def substitute(rng, toks, r):
    out = []
    for kind, i, form in toks:
        if kind == "c" and rng.random() < r:
            form = rng.choice([k for k in range(3) if k != form])
        out.append((kind, i, form))
    return out


def C(b):
    return len(zlib.compress(b, 9))


def ncd(bx, by):
    cx, cy, cxy = C(bx), C(by), C(bx + by)
    return (cxy - min(cx, cy)) / max(cx, cy)


MOVES = "HPCRNOXK"   # hook pain claim proof number offer objection close
NUMS = [3, 7, 9, 12, 15, 19, 24, 30, 49, 59, 99, 128, 199, 299, 399, 999]
NCUM = zipf_cum(len(NUMS))
mrng = random.Random(2930)
# two discourse markers per move, three forms each, four syllables long
MARK = [["".join(mrng.choice(CONS) + mrng.choice(VOWS) for _ in range(4))
         for _ in range(3)] for _ in range(16)]
MARKER = {MARK[j][f]: MOVES[j // 2] for j in range(16) for f in range(3)}


def sales_script(rng, n_sent=12, p_num=0.35, conv=0.6):
    # sentence = [move, marker, form, body tokens, number or None];
    # with chance conv the next move follows the usual genre order
    out, m = [], "H"
    for _ in range(n_sent):
        body = script(rng, 7)[:-1]
        num = NUMS[draw(rng, NCUM)] if rng.random() < p_num else None
        out.append([m, rng.randrange(2), 0, body, num])
        if rng.random() < conv:
            m = MOVES[min(MOVES.index(m) + 1, 7)]
        else:
            m = rng.choice(MOVES)
    return out


def say(sents):
    words = []
    for m, k, form, body, num in sents:
        if m != "?":
            words.append(MARK[2 * MOVES.index(m) + k][form])
        words.append(render(body))
        if num is not None:
            words.append(str(num))
        words[-1] += "."
    return " ".join(words)


def reword(rng, body, r, p_func):
    out = []
    for kind, i, form in body:
        if kind == "c" and rng.random() < r:
            form = rng.choice([x for x in range(3) if x != form])
        elif kind == "f" and rng.random() < p_func:
            i = rng.randrange(30)
        out.append((kind, i, form))
    return out


def light_rewrite(rng, sents, r=0.3):
    return [[m, k, f, reword(rng, b, r, 0.0), n]
            for m, k, f, b, n in sents]


def deep_rewrite(rng, sents, r=0.9):
    # reword, rotate clauses, drop 10 %, insert 15 %, lose 20 % of
    # markers, swap neighbours; the numbers and move order stay
    out = []
    for m, k, form, body, num in sents:
        if rng.random() < 0.1:
            continue
        body = reword(rng, body, r, 0.5)
        cut = rng.randrange(1, len(body))
        mk = "?" if rng.random() < 0.2 else m
        out.append([mk, k, rng.randrange(3), body[cut:] + body[:cut], num])
        if rng.random() < 0.15:
            out.append(["?", 0, 0, script(rng, 7)[:-1], None])
    for i in range(len(out) - 1):
        if rng.random() < 0.15:
            out[i], out[i + 1] = out[i + 1], out[i]
    return out


def fingerprint(text):
    nums, moves = [], []
    for sent in text.split(". "):
        w = sent.rstrip(".").split()
        moves.append(MARKER.get(w[0], "?") if w else "?")
        nums += [x for x in w if x.isdigit()]
    return nums, "".join(m for m in moves if m != "?")


def lcs(a, b):
    prev = [0] * (len(b) + 1)
    for x in a:
        cur = [0]
        for j, y in enumerate(b):
            cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
        prev = cur
    return prev[-1]


def lcs_sim(a, b):
    return lcs(a, b) / max(len(a), len(b), 1)


PR = (1 << 61) - 1
hr = random.Random(97)
AB = [(hr.randrange(1, PR), hr.randrange(PR)) for _ in range(128)]


def sig(text):
    w = text.split()
    g = {zlib.crc32(" ".join(w[i:i + 3]).encode())
         for i in range(len(w) - 2)}
    return [min((a * v + b) % PR for v in g) for a, b in AB]


def lcs_bits(a, b):
    # weighted LCS: total surprisal of numbers matched in order
    prev = [0.0] * (len(b) + 1)
    for x in a:
        cur = [0.0]
        for j, y in enumerate(b):
            cur.append(prev[j] + BITS.get(x, 12.0) if x == y
                       else max(prev[j + 1], cur[j]))
        prev = cur
    return prev[-1]


def typo(rng, s, e):
    return "".join(rng.choice("abdefgiklmnoprstuvz") if
                   c.isalpha() and rng.random() < e else c for c in s)


rng = random.Random(20261113)
null_n, null_f = [], []
for _ in range(300):
    a, b = say(sales_script(rng)), say(sales_script(rng))
    null_n.append(ncd(a.encode(), b.encode()))
    null_f.append(lcs_sim(fingerprint(a)[0], fingerprint(b)[0]))
null_n.sort()
null_f.sort()
t_ncd, t_num = null_n[3], null_f[296]
own_s = [sales_script(rng) for _ in range(40)]
own = [say(s) for s in own_s]
osf = [fingerprint(t)[0] for t in own]
extra = [fingerprint(say(sales_script(rng)))[0] for _ in range(200)]
cnt = {}
for fp in extra:
    for x in fp:
        cnt[x] = cnt.get(x, 0) + 1
tot = math.fsum(cnt.values())
BITS = {x: -math.log2(c / tot) for x, c in cnt.items()}
best = sorted(max(lcs_bits(f, fp) for f in osf) for fp in extra)
t_lib = best[197]
crawl = [("independent", -1, say(sales_script(rng))) for _ in range(340)]
for i in range(10):
    crawl.append(("near copy", i, typo(rng, own[i], 0.02)))
for i in range(20):
    crawl.append(("light", 10 + i, say(light_rewrite(rng, own_s[10 + i]))))
for i in range(30):
    j = i % 40
    crawl.append(("deep", j, say(deep_rewrite(rng, own_s[j]))))
osig = [sig(t) for t in own]
res = {}
for kind, src, text in crawl:
    s = sig(text)
    bands = [i for i in range(40)
             if any(s[b:b + 2] == osig[i][b:b + 2]
                    for b in range(0, 128, 2))]
    st2 = [i for i in bands
           if ncd(own[i].encode(), text.encode()) < t_ncd]
    fp = fingerprint(text)[0]
    sc = [lcs_sim(osf[i], fp) for i in range(40)]
    st3 = [x for x in sc if x > t_num]
    st4 = [i for i in range(40) if lcs_bits(osf[i], fp) > t_lib]
    r = res.setdefault(kind, [0, 0, 0, 0, 0, 0])
    r[0] += 1  # step
    r[1] += bool(bands)  # step
    r[2] += bool(st2)  # step
    r[3] += bool(st3)  # step
    r[4] += bool(st4)  # step
    r[5] += bool(st2 or st4)  # step
print("thresholds: NCD < %.4f, NUM pair > %.4f, NUM bits > %.2f"
      % (t_ncd, t_num, t_lib))
print("type         pages  LSH  NCD  NUM pair  NUM bits  to review")
for kind in ("near copy", "light", "deep", "independent"):
    r = res[kind]
    print("%-12s %4d  %4d %4d    %4d      %4d      %4d"
          % ((kind,) + tuple(r)))

Single source and citation

Fan, Y. (2026). Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts (Version 1.0.0), Chapter 8: Closing the Loop: Weekly Monitoring for Rewritten Copies, De-duplicating Direct-Answer Pages and a Pre-registered Test. 5D Library. DOI pending

DOI pending · monograph page goes live after publication · 5DL-B-293 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026293,
  author    = {Fan, Yafu},
  title     = {Normalised Compression Distance and Kolmogorov Complexity: Detecting Rewritten Copies and Clustering Families of Shared-Origin Sales Scripts},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

Other chapter answers in this book