What does Sparse Probes, Hidden Discounts conclude, in numbers, about “Closing the Loop: a Monthly Mystery-Shopper Probe Schedule, a Compliance Checklist and a Pre-Registered Test”?

Answer: Chapter 8 gives a six-step monthly probe procedure, a seven-item compliance checklist, the Sparse Probe Reconstructor and a pre-registered test. Marked boxes separate published evidence, runnable Python calculations (18 programs), classical sources and the author's hypotheses.

Scope of the numbers: Its items, amounts, costs and rivals are illustrative and all calculations are generated by its programs; it is a general business reference, not a legal opinion. Marked boxes separate published evidence, runnable Python calculations (18 programs), classical sources and the author's hypotheses.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python only

Sparse Probe Reconstructor (5D Library B-242), illustrative demo
plan: n = 120 cells, expected k = 5, c = 1.5
  fitting quotes m = ceil(c k ln(n/k)) = 24, hold-outs h = 3
  census would need 120 quotes; plan uses 27
  basket sizes: smallest 50, largest 68 items
  design check: empty columns 0, twin column pairs 0
demo competitor: 5 invented hidden discounts
recovered cells (exact simplex in fractions):
  item  15  discount 39 yuan  (truth 39)
  item  17  discount 26 yuan  (truth 26)
  item  86  discount 47 yuan  (truth 47)
  item 105  discount 6 yuan  (truth 6)
  item 116  discount 50 yuan  (truth 50)
hold-out quotes predicted to the yuan: 3 of 3
status: CANDIDATE - a person reviews it before it enters the price table
boundary: public prices and conditions only; real identity;
  no staff questioning; no scraping behind logins; no sharing
Show the full program
# Chapter 8. p81: the Sparse Probe Reconstructor's demo, line for line.
# The functions below are copied from distill/tool.py (which adds a
# command line and JSON output): exact two-phase simplex in fractions,
# the plan rule m = ceil(c k ln(n/k)) with c = 1.5, the design check,
# the robust variant, and hold-out checks. Demo: n = 120 listed items,
# 5 invented hidden discounts, 24 fitting quotes and 3 hold-outs,
# basket design seed 20261181. Illustrative only.
import math
import random
from fractions import Fraction as Fr


def esum(v):
    s = Fr(0)
    for t in v:
        s += t
    return s


def lp_min(A, b, c, exact=True):
    # min c.x subject to A x = b, x >= 0: two-phase tableau simplex
    m, n = len(A), len(A[0])
    cv = Fr if exact else float
    add = esum if exact else math.fsum
    tol = 0 if exact else 1e-9
    big = 1 + max(abs(v) for v in b)
    d = [Fr(1, 2 ** (30 + j)) if exact else
         1e-7 * big * (0.5 + (0.6180339887 * (j + 1)) % 1.0)
         for j in range(n)]
    T = []
    for i in range(m):
        s = -1 if b[i] < 0 else 1
        row = [cv(s * v) for v in A[i]] + [cv(0)] * (m + 1)
        row[n + i] = cv(1)
        row[-1] = cv(abs(b[i])) + s * add([A[i][j] * d[j]
                                           for j in range(n)])
        T.append(row)
    basis = [n + i for i in range(m)]
    W = n + m

    def pivot(r, q, z):
        pv = T[r][q]
        T[r] = [v / pv for v in T[r]]
        pr = T[r]
        nz = [j for j in range(W + 1) if pr[j] != 0]
        for row in T + [z]:
            f = row[q]
            if row is not pr and f != 0:
                for j in nz:
                    row[j] -= f * pr[j]
        basis[r] = q

    def run(z, stop):
        while not (stop and -z[W] <= tol):
            q, best = -1, -1e-7
            for j in range(n):
                if exact and z[j] < 0:
                    q = j
                    break
                if not exact and z[j] < best:
                    q, best = j, z[j]
            if q < 0:
                return
            r, rt = -1, None
            for i in range(m):
                if T[i][q] > tol:
                    t = T[i][W] / T[i][q]
                    if rt is None or t < rt:
                        r, rt = i, t
            pivot(r, q, z)

    z = [cv(0)] * (W + 1)
    for j in list(range(n)) + [W]:
        z[j] = -add([T[i][j] for i in range(m)])
    run(z, True)
    if -z[W] > (0 if exact else 1e-6 * big):
        return None
    z = [cv(v) for v in c] + [cv(0)] * (m + 1)
    for i in range(m):
        if basis[i] < n and z[basis[i]] != 0:
            f = z[basis[i]]
            z = [z[j] - f * T[i][j] for j in range(W + 1)]
    run(z, False)
    x = [cv(0)] * n
    for i in range(m):
        if basis[i] < n:
            x[basis[i]] = add([T[i][n + j] * abs(b[j])
                               for j in range(m)])
    return x


def plan_m(n, k, c):
    return max(1, math.ceil(c * k * math.log(n / k)))


def design(n, rows, seed):
    rng = random.Random()
    rng.seed(seed)
    return [[1 if rng.random() < 0.5 else 0 for _ in range(n)]
            for _ in range(rows)]


def design_check(A, n):
    cols = [tuple(r[j] for r in A) for j in range(n)]
    zero = [j for j in range(n) if not any(cols[j])]
    seen, twins = {}, []
    for j in range(n):
        if any(cols[j]):
            if cols[j] in seen:
                twins.append((seen[cols[j]], j))
            else:
                seen[cols[j]] = j
    return zero, twins


def recover(A, y, robust=False):
    n, m = len(A[0]), len(A)
    if not robust:
        x = lp_min(A, y, [1] * n)
        return x, [Fr(0)] * m
    AE = [A[i] + [1 if t == i else 0 for t in range(m)]
          + [-1 if t == i else 0 for t in range(m)] for i in range(m)]
    z = lp_min(AE, y, [1] * (n + 2 * m))
    return z[:n], [z[n + i] - z[n + m + i] for i in range(m)]


def predict(A, x):
    return [esum([x[j] for j in range(len(x)) if r[j]]) for r in A]


def run_round(A, y, B, yb, robust=False):
    x, e = recover(A, y, robust)
    out = {"feasible": x is not None}
    if x is None:
        return out
    out["cells"] = [[j, str(v)] for j, v in enumerate(x) if v != 0]
    out["spoiled_quotes"] = [i for i, v in enumerate(e) if v != 0]
    pred = predict(B, x)
    out["holdout_pass"] = len([1 for p, t in zip(pred, yb) if p == t])
    out["holdout_total"] = len(yb)
    out["candidate"] = (out["holdout_pass"] == len(yb) and len(yb) > 0)
    return out


def demo_data(n, k, seed):
    rng = random.Random()
    rng.seed(seed + 1)
    x = [0] * n
    for j in rng.sample(range(n), k):
        x[j] = rng.randint(5, 60)
    return x


def demo(n=120, k=5, c=1.5, h=3, seed=20261181):
    m = plan_m(n, k, c)
    A = design(n, m + h, seed)
    zero, twins = design_check(A[:m], n)
    x = demo_data(n, k, seed)
    y = [int(v) for v in predict(A, x)]
    res = run_round(A[:m], y[:m], A[m:], y[m:])
    sizes = [len([v for v in r if v]) for r in A]
    lines = [
        "Sparse Probe Reconstructor (5D Library B-242), illustrative demo",
        "plan: n = %d cells, expected k = %d, c = %.1f" % (n, k, c),
        "  fitting quotes m = ceil(c k ln(n/k)) = %d, hold-outs h = %d"
        % (m, h),
        "  census would need %d quotes; plan uses %d" % (n, m + h),
        "  basket sizes: smallest %d, largest %d items"
        % (min(sizes), max(sizes)),
        "  design check: empty columns %d, twin column pairs %d"
        % (len(zero), len(twins)),
        "demo competitor: %d invented hidden discounts" % k,
        "recovered cells (exact simplex in fractions):"]
    for j, v in res["cells"]:
        lines.append("  item %3d  discount %s yuan  (truth %d)"
                     % (j, v, x[j]))
    lines += [
        "hold-out quotes predicted to the yuan: %d of %d"
        % (res["holdout_pass"], res["holdout_total"]),
        "status: %s" % ("CANDIDATE - a person reviews it before it"
                        " enters the price table" if res["candidate"]
                        else "REJECT - add quotes or census"),
        "boundary: public prices and conditions only; real identity;",
        "  no staff questioning; no scraping behind logins; no sharing"]
    return lines, {"plan": {"cells": n, "k": k, "c": c, "m": m,
                            "holdout": h},
                   "design": {"empty": zero, "twins": twins},
                   "round": res}


lines, _ = demo()
for ln in lines:
    print(ln)

Single source and citation

Fan, Y. (2026). Compressed Sensing and L1 Minimisation: Sparse Mystery-Shopper Probes and the Exact Recovery of Competitors' Hidden-Discount Matrices (Version 1.0.0), Chapter 8: Closing the Loop: a Monthly Mystery-Shopper Probe Schedule, a Compliance Checklist and a Pre-Registered Test. 5D Library. DOI pending

DOI pending · monograph page goes live after publication · 5DL-B-242 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026242,
  author    = {Fan, Yafu},
  title     = {Compressed Sensing and L1 Minimisation: Sparse Mystery-Shopper Probes and the Exact Recovery of Competitors' Hidden-Discount Matrices},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  note      = {DOI pending}
}

Other chapter answers in this book