关于「p 值操纵、研究者自由度与分岔花园」,《可证伪性与可重复性危机》给出的定量结论是什么?
结论:在真实效应为零的数据中,挑选因变量、追加样本、加入协变量与取舍条件四种做法,各自把假阳性率从 5% 抬高到约 8% 到 13%,合在一起达到约六成(原文 60.7%;本书重做,性别按效应编码为 57.9%,按 0/1 编码为 61.2%)。可选停止越频繁,假阳性越多。\(p\) 曲线的形状可以提示一组文献是否被操纵。格尔曼与洛肯强调,即使没有故意钓鱼,依赖数据的分析选择也会让 \(p\) 值失去意义。下一章讨论另一层筛选:什么样的结果会被发表。
数据口径:本书不是医疗、心理或投资建议。 书中以标注框区分已发表的证据、可运行的 Python 计算(14 段程序,全部核对)、原典与作者的假说。
对照数据表
下表数字由本章 Python 程序原样打印,可重新运行复现。 程序只用 Python 标准库与结论吻合的打印值: 57.9, 60.7, 61.2
| 15,000 simulated studies, no true effect | this run: p<.10 | p<.05 | p<.01 | paper: .10 | .05 | .01 |
| baseline: one DV, one test | 10.1 | 5.1 | 0.9 | | (10.0, 5.0, 1.0 expected) | |
| A: two DVs (r = .5) or their mean | 18.7 | 9.8 | 2.1 | 17.8 | 9.5 | 2.2 |
| B: add 10 per cell if not significant | 14.9 | 7.9 | 1.5 | 14.5 | 7.7 | 1.6 |
| C: gender covariate or interaction | 21.0 | 11.1 | 2.4 | 21.6 | 11.7 | 2.7 |
| D: drop (or keep) one of 3 conditions | 23.1 | 12.4 | 2.7 | 23.2 | 12.6 | 2.8 |
| A + B | 26.9 | 14.6 | 3.3 | 26.0 | 14.4 | 3.3 |
| A + B + C | 48.7 | 29.1 | 7.5 | 50.9 | 30.9 | 8.4 |
| A + B + C + D | 79.7 | 57.9 | 19.6 | 81.5 | 60.7 | 21.5 |
| same samples, gender coded 0/1 instead of +-0.5: | |||||
| C, gender coded 0/1 | 22.6 | 12.4 | 2.7 | 21.6 | 11.7 | 2.7 |
| A + B + C, gender coded 0/1 | 51.4 | 31.6 | 8.6 | 50.9 | 30.9 | 8.4 |
| A + B + C + D, gender coded 0/1 | 81.9 | 61.2 | 21.8 | 81.5 | 60.7 | 21.5 |
查看完整验算程序
# Researcher degrees of freedom: a Monte Carlo re-run of Table 1 of Simmons, Nelson &
# Simonsohn (2011). Every effect is truly ZERO: all observations are independent N(0, 1)
# noise. Baseline: two conditions, 20 per cell, one t test. A: two dependent variables
# correlated r = .5; test DV1, DV2 or their mean. B: test at 20 per cell; if not
# significant add 10 per cell and test again. C: t test, ANCOVA with gender, or gender x
# condition (condition OR interaction counts); each person is female with probability .5.
# The paper does not state how gender is coded; in the interaction model this matters:
# effect coding (+-0.5) tests the AVERAGE condition effect, 0/1 coding tests it within
# one gender, a different test. Both are run on the same samples. D: three conditions;
# any of the three pairwise tests or the linear trend (-1, 0, 1). A result "counts" if
# ANY analysis allowed in that situation reaches the threshold. Critical t values by the
# Cornish-Fisher expansion (accurate to about 1e-4 for df > 30).
import math, random
from statistics import NormalDist
def t_crit(alpha, df):
z = NormalDist().inv_cdf(1 - alpha / 2)
return (z + (z ** 3 + z) / (4 * df)
+ (5 * z ** 5 + 16 * z ** 3 + 3 * z) / (96 * df ** 2)
+ (3 * z ** 7 + 19 * z ** 5 + 17 * z ** 3 - 15 * z) / (384 * df ** 3))
CRIT = {df: [t_crit(a, df) for a in (0.10, 0.05, 0.01)] for df in range(30, 90)}
def ols_level(cells, tests):
# cells: (n, sum y, sum y^2, x); level = thresholds (.10/.05/.01) passed by any
# tested beta
p = len(cells[0][3])
A = [[sum(n * x[i] * x[j] for n, _, _, x in cells) for j in range(p)]
for i in range(p)]
v = [sum(s * x[i] for _, s, _, x in cells) for i in range(p)]
M = [row + [float(i == j) for j in range(p)] for i, row in enumerate(A)]
for i in range(p): # Gauss-Jordan inverse of X'X
if abs(M[i][i]) < 1e-12:
return 0
M[i] = [e / M[i][i] for e in M[i]]
for k in range(p):
if k != i:
M[k] = [a - M[k][i] * b for a, b in zip(M[k], M[i])]
inv = [row[p:] for row in M]
b = [sum(inv[i][j] * v[j] for j in range(p)) for i in range(p)]
df = sum(c[0] for c in cells) - p
s2 = (sum(c[2] for c in cells) - sum(bi * vi for bi, vi in zip(b, v))) / df
return max(sum(abs(b[j]) / math.sqrt(s2 * inv[j][j]) > c for c in CRIT[df])
for j in tests)
DESIGNS = { # condition codes used by each comparison
"01": {0: -0.5, 1: 0.5}, "02": {0: -0.5, 2: 0.5}, "12": {1: -0.5, 2: 0.5},
"trend": {0: -1.0, 1: 0.0, 2: 1.0}}
def one_sample(rng):
data = [] # per condition: list of (gender, y1, y2)
for _ in range(3):
rows = []
for _ in range(30):
z1, z2 = rng.gauss(0, 1), rng.gauss(0, 1)
g = 0.5 if rng.random() < 0.5 else -0.5
rows.append((g, z1, 0.5 * z1 + math.sqrt(0.75) * z2))
data.append(rows)
levels = {}
for stage in (20, 30):
for dv in (0, 1, 2):
cell = {}
for k in range(3):
for g, y1, y2 in data[k][:stage]:
y = (y1, y2, (y1 + y2) / 2)[dv]
c = cell.setdefault((k, g), [0, 0.0, 0.0])
c[0] += 1; c[1] += y; c[2] += y * y
for dname, code in DESIGNS.items():
cs = [(c[0], c[1], c[2], code[k], g) for (k, g), c in cell.items()
if k in code]
models = {
"plain": ([(n, s, q, (1.0, x)) for n, s, q, x, g in cs], (1,)),
"cov": ([(n, s, q, (1.0, x, g)) for n, s, q, x, g in cs], (1,)),
"int": ([(n, s, q, (1.0, x, g, x * g)) for n, s, q, x, g in cs],
(1, 3)),
"int01": ([(n, s, q, (1.0, x, g + 0.5, x * (g + 0.5)))
for n, s, q, x, g in cs], (1, 3))}
for mname, (cells, tests) in models.items():
levels[(stage, dv, dname, mname)] = ols_level(cells, tests)
return levels
D3 = tuple(DESIGNS)
S2, DV3, C3, C01 = (20, 30), (0, 1, 2), ("plain", "cov", "int"), ("plain", "cov", "int01")
ROWS = [ # (label, stages, DVs, comparisons, models, Table 1: p < .10 / .05 / .01)
("baseline: one DV, one test", (20,), (0,), ("01",), ("plain",),
" (10.0, 5.0, 1.0 expected)"),
("A: two DVs (r = .5) or their mean", (20,), DV3, ("01",), ("plain",),
"17.8 9.5 2.2"),
("B: add 10 per cell if not significant", S2, (0,), ("01",), ("plain",),
"14.5 7.7 1.6"),
("C: gender covariate or interaction", (20,), (0,), ("01",), C3,
"21.6 11.7 2.7"),
("D: drop (or keep) one of 3 conditions", (20,), (0,), D3, ("plain",),
"23.2 12.6 2.8"),
("A + B", S2, DV3, ("01",), ("plain",), "26.0 14.4 3.3"),
("A + B + C", S2, DV3, ("01",), C3, "50.9 30.9 8.4"),
("A + B + C + D", S2, DV3, D3, C3, "81.5 60.7 21.5"),
("C, gender coded 0/1", (20,), (0,), ("01",), C01, "21.6 11.7 2.7"),
("A + B + C, gender coded 0/1", S2, DV3, ("01",), C01, "50.9 30.9 8.4"),
("A + B + C + D, gender coded 0/1", S2, DV3, D3, C01, "81.5 60.7 21.5")]
def run(n_sims=15000, seed=2011):
rng = random.Random(seed)
hits = [[0, 0, 0] for _ in ROWS]
for _ in range(n_sims):
lv = one_sample(rng)
for r, (_, st, dvs, ds, ms, _) in enumerate(ROWS):
best = max(lv[(s, v, d, m)] for s in st for v in dvs
for d in ds for m in ms)
for k in range(best):
hits[r][k] += 1
return [[100 * h / n_sims for h in row] for row in hits]
if __name__ == "__main__":
res = run()
print("15,000 simulated studies, no true effect this run: p<.10 p<.05 p<.01"
" | paper: .10 .05 .01")
for k, (row, r) in enumerate(zip(ROWS, res)):
if k == 8:
print("same samples, gender coded 0/1 instead of +-0.5:")
print(f"{row[0]:40s} {r[0]:5.1f} {r[1]:5.1f} {r[2]:5.1f} | {row[5]}")唯一数据源与引用
范雅福(Fan, Y.). (2026). 《科学方法论、可证伪性与可重复性危机的数理统计》第 3 章「p 值操纵、研究者自由度与分岔花园」(版本 1.0.0). 5D Library. https://doi.org/10.5281/zenodo.23782104
DOI: 10.5281/zenodo.23782104 · 阅读本章所在专著 · 5DL-B-033 · v1.0.0 · CC-BY-NC-ND-4.0
BibTeX
@book{fan2026033,
author = {Fan, Yafu},
title = {Scientific Methodology, Falsifiability and the Mathematical Statistics of the Replication Crisis},
year = {2026},
publisher = {5D Library},
edition = {1.0.0},
doi = {10.5281/zenodo.23782104},
url = {https://doi.org/10.5281/zenodo.23782104}
}