What does Information Geometry and the Natural Gradient conclude, in numbers, about “Practice and a Pre-Registered Test”?

Answer: Chapter 8 gives a four-step checklist and a pre-registered computational benchmark: on a stratified suite of synthetic tasks frozen by an independent third party, natural-gradient optimisation (exact or K-FAC) is compared with Adam on wall-clock time to a target loss; the single primary outcome is the mean over tasks of the base-2 logarithm of the time ratio, the smallest effect of interest log₂1.25, 393 paired tasks for 90 % power and 470 planned after allowing for invalid tasks, with the decision rule written as code. Marked boxes separate published evidence, runnable Python calculations (16 programs), classical sources and the author's models and hypotheses; all data in the book are synthetic or simulated and are no guarantee of the performance of any particular machine-learning system.

中文版

Data table

The numbers below are printed verbatim by the chapter's Python program and can be reproduced by running it. standard-library Python onlyprinted values that match the answer: 1.25, 393, 470, 90

pilot (40 tasks): mean D -2.78 (NG faster if < 0), sd 1.96,
   range -6.61 to +0.36; Adam missed the target on 4 task(s) (time set to 2 x cap)
SESOI log2(1.25) = 0.322; 90 % power, two-sided alpha 0.05: n = 392.5 -> 393 tasks
   simulation (2,000 resampled suites of 393, true mean D = -SESOI): power 0.906
   allow 15 % lost to infrastructure faults: generate 470 (94 in each of 5 strata)
sensitivity: tasks needed (before the 15 % allowance)
   sd of D:     0.98   1.96   2.94   3.93
   n:            100    393    881   1565
blinded re-estimation after 25 % of tasks: sd of D -> tasks to generate
   1.5 -> 470  2.0 -> 480  2.5 -> 750  3.0 -> 1,080  4.0 -> 1,920
Show the full program
# Sample size for the Chapter 8 pre-registered benchmark. Unit = a synthetic task drawn
# by the third party; both optimisers run on it (paired). Primary outcome per task:
# D = log2(time to target, natural gradient / time to target, Adam). Estimand: mean D
# over the task population (log2 of the geometric-mean time ratio). SESOI log2(1.25).
# PILOT (planning assumption only, not the trial suite): 40 logistic-regression tasks,
# n = 40, d = 4, kappa log-uniform on [1, 10^4], random rotation; "time" = gradient-
# evaluation units (Newton/NG step = 1 + d + d^2/(3n)); Adam: best of 3 steps, cap 500.
# Last part: the registered blinded re-estimation rule (sd of D from the first 25 % of
# tasks; sd is unchanged by swapping the arm labels; n may only go up, cap 2,000).
import math, random, statistics

NZ = statistics.NormalDist(); Z = NZ.inv_cdf

def task(rng, n=40, d=4):
    kappa = 10 ** rng.uniform(0, 4); ev = [kappa ** (-i / (d - 1)) for i in range(d)]
    Q = []
    while len(Q) < d:
        v = [rng.gauss(0, 1) for _ in range(d)]
        for q in Q:
            c = sum(a * b for a, b in zip(v, q)); v = [a - c * b for a, b in zip(v, q)]
        s = math.sqrt(sum(a * a for a in v)); Q.append([a / s for a in v])
    X = []
    for _ in range(n):
        z = [rng.gauss(0, 1) * math.sqrt(e) for e in ev]
        X.append([sum(Q[i][j] * z[j] for j in range(d)) for i in range(d)])
    w = [sum(Q[i][j] / math.sqrt(ev[j]) for j in range(d)) / d for i in range(d)]
    y = [int(rng.random() < 1 / (1 + math.exp(-sum(a * b for a, b in zip(w, x)))))
         for x in X]
    return X, y

def lgf(X, y, w, hess=False):
    d, n = len(w), len(X); L, g = 0.0, [0.0] * d; H = [[0.0] * d for _ in range(d)]
    for x, t in zip(X, y):
        z = sum(a * b for a, b in zip(w, x))
        p = 1 / (1 + math.exp(-z)) if z >= 0 else math.exp(z) / (1 + math.exp(z))
        L += math.log1p(math.exp(-abs(z))) + max(z, 0) - t * z
        for i in range(d):
            g[i] += (p - t) * x[i] / n
            if hess:
                for j in range(d):
                    H[i][j] += p * (1 - p) * x[i] * x[j] / n
    return L / n, g, H

def newton(X, y, w):
    L, g, H = lgf(X, y, w, True); d = len(w)
    M = [H[i][:] + [g[i]] for i in range(d)]
    for i in range(d):
        M[i][i] += 1e-8
    for c in range(d):
        for r in range(c + 1, d):
            f = M[r][c] / M[c][c]; M[r] = [a - f * b for a, b in zip(M[r], M[c])]
    s = [0.0] * d
    for r in range(d - 1, -1, -1):
        s[r] = (M[r][d] - sum(M[r][k] * s[k] for k in range(r + 1, d))) / M[r][r]
    return [a - b for a, b in zip(w, s)], L

def pilot_D(seed):
    rng = random.Random(seed); X, y = task(rng); d, n = len(X[0]), len(X)
    w = [0.0] * d
    for _ in range(30):
        w, _ = newton(X, y, w)
    target = lgf(X, y, w)[0] + 1e-3; w = [0.0] * d; k_ng = 0
    while lgf(X, y, w)[0] > target:
        w, _ = newton(X, y, w); k_ng += 1
    best = None
    for lr in (0.03, 0.1, 0.3):
        w = [0.0] * d; m = [0.0] * d; v = [0.0] * d
        for k in range(1, 501):
            L, g, _ = lgf(X, y, w)
            if L <= target:
                best = k - 1 if best is None else min(best, k - 1); break
            m = [0.9 * a + 0.1 * b for a, b in zip(m, g)]
            v = [0.999 * a + 0.001 * b * b for a, b in zip(v, g)]
            w = [a - lr * (b / (1 - 0.9 ** k)) / (math.sqrt(c / (1 - 0.999 ** k)) + 1e-8)
                 for a, b, c in zip(w, m, v)]
    t_ng = k_ng * (1 + d + d * d / (3 * n)); t_ad = best if best is not None else 1000
    return math.log2(max(t_ng, 1) / max(t_ad, 1)), best is None

NP = 40
Ds = [pilot_D(s) for s in range(NP)]
D = [a for a, _ in Ds]; sd = statistics.stdev(D)
print(f"pilot ({NP} tasks): mean D {statistics.fmean(D):+.2f} (NG faster if < 0),"
      f" sd {sd:.2f},"
      f"\n   range {min(D):+.2f} to {max(D):+.2f}; Adam missed the target on"
      f" {sum(f for _, f in Ds)} task(s) (time set to 2 x cap)")
delta = math.log2(1.25); za, zb = Z(0.975), Z(0.90)
n0 = ((za + zb) * sd / delta) ** 2 + za * za / 2
n_t = math.ceil(n0)
print(f"SESOI log2(1.25) = {delta:.3f}; 90 % power, two-sided alpha 0.05:"
      f" n = {n0:.1f} -> {n_t} tasks")
rng = random.Random(15); hits = 0; R = 2000
cen = [a - statistics.fmean(D) - delta for a in D]       # true mean D = -SESOI
for _ in range(R):
    s = [rng.choice(cen) for _ in range(n_t)]
    m, se = statistics.fmean(s), statistics.stdev(s) / math.sqrt(n_t)
    hits += m + 1.96 * se < 0                              # "NG faster" shown
print(f"   simulation ({R:,} resampled suites of {n_t}, true mean D = -SESOI):"
      f" power {hits / R:.3f}")
plan = math.ceil(n_t / 0.85 / 10) * 10
print(f"   allow 15 % lost to infrastructure faults: generate {plan}"
      f" ({plan // 5} in each of 5 strata)")
print("sensitivity: tasks needed (before the 15 % allowance)")
SDS = (0.5 * sd, sd, 1.5 * sd, 2 * sd)
print("   sd of D:    " + "  ".join(f"{s:5.2f}" for s in SDS))
need = lambda s: math.ceil(((za + zb) * s / delta) ** 2 + za * za / 2)
print("   n:          " + "  ".join(f"{need(s):5d}" for s in SDS))

def reestimate(sd_blind, n_plan=plan, cap=2000):     # registered rule, never decreases
    n_new = math.ceil(need(sd_blind) / 0.85 / 10) * 10
    return min(cap, max(n_plan, n_new))
print("blinded re-estimation after 25 % of tasks: sd of D -> tasks to generate")
SD2 = (1.5, 2.0, 2.5, 3.0, 4.0)
print("   " + "  ".join(f"{s:.1f} -> {reestimate(s):,}" for s in SD2))

Single source and citation

Fan, Y. (2026). Information Geometry, the Fisher Information Metric and Natural-Gradient Optimisation on Statistical Manifolds (Version 1.0.0), Chapter 8: Practice and a Pre-Registered Test. 5D Library. https://doi.org/10.5281/zenodo.23836268

DOI: 10.5281/zenodo.23836268 · Read the monograph · 5DL-B-049 · v1.0.0 · CC-BY-NC-ND-4.0

BibTeX
@book{fan2026049,
  author    = {Fan, Yafu},
  title     = {Information Geometry, the Fisher Information Metric and Natural-Gradient Optimisation on Statistical Manifolds},
  year      = {2026},
  publisher = {5D Library},
  edition   = {1.0.0},
  doi       = {10.5281/zenodo.23836268},
  url       = {https://doi.org/10.5281/zenodo.23836268}
}

Other chapter answers in this book