diff --git a/.github/workflows/site-provenance.yml b/.github/workflows/site-provenance.yml new file mode 100644 index 0000000..0142f8f --- /dev/null +++ b/.github/workflows/site-provenance.yml @@ -0,0 +1,24 @@ +name: Site provenance + +# The validation charts are generated from the papers' committed data, and +# make_charts.py --check exists to prove the page has not drifted from it. +# Nothing ran it, so it silently stopped holding: #89 added three hand-authored +# charts and the gate had been failing on main ever since. Run it in CI. +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + charts: + name: validation/index.html matches its sources + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Charts are regenerable from committed data + run: python3 validation/figures/make_charts.py --check diff --git a/validation/figures/chart_data.json b/validation/figures/chart_data.json index f62f2b4..6f98890 100644 --- a/validation/figures/chart_data.json +++ b/validation/figures/chart_data.json @@ -21,6 +21,29 @@ ] }, + "obr_computed_share": { + "source": "PolicyEngine/obr-macroeconomic-model docs/calibration_scorecard.md (March 2026 EFO vintage), cross-checked against papers/obr-macro/sections/validation.tex", + "total": 21, + "computed": 11, + "passthrough": 10, + "bands_note": "bands: rates ±1.0pp · net balances ±1.5% of GDP · levels ≤10% MAPE", + "_grades_comment": "The 11 computed variables graded against the bands above. 'identity' is employment, where both inputs are themselves passthrough, so the accounting identity closes trivially and the pass carries no information.", + "grades": [ + { "key": "fair", "label": "fair", "count": 3, "series": 1, "in_band": true, + "examples": ["real GDP 5.75 per cent", "consumption 9.56 per cent", "trade balance 0.70 per cent of GDP"], + "source": "calibration_scorecard.md, free-running MAPE column" }, + { "key": "identity", "label": "identity", "count": 1, "series": 2, "in_band": true, "trivial": true, + "examples": ["employment"], + "source": "calibration_scorecard.md; employment closes from two passthrough inputs" }, + { "key": "poor", "label": "poor", "count": 5, "series": 3, "in_band": false, + "examples": [], + "source": "calibration_scorecard.md, variables outside band but within an order of magnitude" }, + { "key": "off", "label": "off", "count": 2, "series": 3, "in_band": false, + "examples": ["company profits 79.80 per cent", "the current account 4.17 per cent of GDP"], + "source": "calibration_scorecard.md, worst two computed variables" } + ] + }, + "frbus_residuals": { "source": "papers/frb-us/sections/validation.tex, Tables tab:tracking and tab:refnoise", "rows": [ diff --git a/validation/figures/make_charts.py b/validation/figures/make_charts.py index c2d1613..62e9770 100644 --- a/validation/figures/make_charts.py +++ b/validation/figures/make_charts.py @@ -112,6 +112,37 @@ def load_transcribed(): return json.loads((HERE / "chart_data.json").read_text()) +def load_rolling_eval(): + """Expanding-window forecast skill against a random walk with drift. + + Source: papers/boe-svar/figures/rolling_evaluation.json, which stores, per + horizon, ``relative_rmse_vs_drift_by_variable`` (model RMSE / benchmark + RMSE; below 1 means the model wins) and ``dm_pvalue_vs_drift_by_variable`` + (Diebold-Mariano p-value for that difference). + + The drift benchmark is used rather than the plain random walk because it is + the harder of the two: a drifting series makes a no-change forecast look + worse than it is, and the model's advantage shrinks against drift. + """ + d = json.loads( + (ROOT / "papers" / "boe-svar" / "figures" / "rolling_evaluation.json").read_text() + ) + horizons = sorted(d["horizons"], key=lambda h: h["horizon"]) + ratio = {h["horizon"]: h["relative_rmse_vs_drift_by_variable"] for h in horizons} + pval = {h["horizon"]: h["dm_pvalue_vs_drift_by_variable"] for h in horizons} + return [h["horizon"] for h in horizons], ratio, pval, d + + +# Descriptions are read aloud by screen readers, where digits for small counts +# scan worse than words. +WORDS = {1: "one", 2: "two", 3: "three", 4: "four", 5: "five", + 6: "six", 7: "seven", 8: "eight"} + + +def word(k): + return WORDS.get(k, str(k)) + + # ---------------------------------------------------------------- chart builders def svg_open(view_w, view_h, chart_id, title, desc): @@ -496,14 +527,253 @@ def chart_svar_fan(): return "\n".join(out) +def chart_obr_computed_share(): + """Two stacked bars: how much of the OBR scorecard the model actually computes. + + The point of the chart is that a headline "within band" rate is meaningless + until the passthrough variables — held at the OBR published value, so scoring + zero error trivially — are separated out from the ones the model computes. + """ + d = load_transcribed()["obr_computed_share"] + total = d["total"] + computed, passthrough = d["computed"], d["passthrough"] + grades = d["grades"] + + W, H = 760, 270 + x0, x1 = 64.0, 724.0 + span = x1 - x0 + per_var = span / total + + in_band = sum(g["count"] for g in grades if g["in_band"]) + trivial = sum(g["count"] for g in grades if g.get("trivial")) + off = next(g for g in grades if g["key"] == "off") + + desc = ( + f"Two stacked bars. Of {total} headline variables in the OBR emulator " + f"calibration scorecard, {computed} are actually computed by the model and " + f"{passthrough} are passthrough, held at the OBR published value and therefore " + f"scoring zero error trivially. Of the {computed} computed, " + + ", ".join(f"{g['count']} are {g['label']}" for g in grades) + + f". {in_band} of the {computed}, or {100 * in_band / computed:.0f} per cent, land " + f"within band, and {word(trivial)} of those is a trivial accounting identity, so only " + f"{in_band - trivial} non-trivial computed variables are in band. The worst are " + + " and ".join(off["examples"]) + + "." + ) + + out = svg_open(W, H, "obr-computed-share", + "How much of the OBR emulator scorecard the model actually computes", desc) + + # Row 1 — computed vs passthrough, out of the full scorecard. + split = x0 + computed * per_var + out.append(f'{total} headline scorecard variables') + out.append(f'') + out.append(f'') + out.append(f'' + f'{computed} computed') + out.append(f'' + f'{passthrough} passthrough — held at the OBR value') + out.append(f'') + out.append(f'') + + # Row 2 — the computed variables broken down by grade, on the same scale, so + # the second bar reads as a zoom into the first bar's left-hand segment. + out.append(f'of which, the {computed} the model ' + f'computes') + x = x0 + for g in grades: + w = g["count"] * per_var + dash = ' stroke="currentColor" stroke-dasharray="3 2" stroke-width="1"' if g["key"] == "off" else "" + out.append(f'') + out.append(f'' + f'{esc(g["label"])} {g["count"]}') + x += w + + out.append(f'Only {in_band} of the {computed} computed ' + f'variables land within band — {100 * in_band / computed:.0f}% — and ' + f'{word(trivial)} of those {word(in_band)} is a trivial identity.') + out.append(f'{esc(d["bands_note"])}') + out.append("") + return "\n".join(out) + + +# Display order and labels for the skill matrix. Ordered roughly best to worst so +# the eye can read down the column; the ordering is presentational, the numbers +# come from rolling_evaluation.json. +SKILL_ROWS = [ + ("bank_rate", "Bank Rate"), + ("cpisa", "UK CPI"), + ("world_cpi", "World CPI"), + ("oil_price", "Oil price"), + ("cpi_energy", "CPI energy"), + ("uk_gdp", "UK real GDP"), + ("world_gdp", "World GDP"), + ("eri", "Exchange rate"), +] + +SIGNIF = 0.05 + + + +def chart_svar_skill_all(): + horizons, ratio, pval, meta = load_rolling_eval() + W, H = 760, 404 + x_one, per_unit = 370.8, 883.0 # value 1.0 at x_one + xs = lambda v: x_one + (v - 1.0) * per_unit + + last = horizons[-1] + best = min(SKILL_ROWS, key=lambda r: ratio[last][r[0]]) + worst = max(SKILL_ROWS, key=lambda r: ratio[last][r[0]]) + n_win_1 = sum(1 for k, _ in SKILL_ROWS if ratio[horizons[0]][k] < 1) + n_win_last = sum(1 for k, _ in SKILL_ROWS if ratio[last][k] < 1) + + desc = ( + f"Dot matrix of forecast error relative to a random walk with drift, " + f"{word(len(SKILL_ROWS))} variables by {word(len(horizons))} quarterly horizons, from " + f"{meta['origins']} expanding-window origins. A ratio below 1.0 means the model " + f"beats the benchmark; filled dots mark differences significant at 5 per cent by a " + f"Diebold-Mariano test, hollow dots differences that are not statistically " + f"distinguishable. " + + "; ".join( + f"{label} runs {ratio[horizons[0]][key]:.2f} at one quarter to " + f"{ratio[last][key]:.2f} at {last}" + for key, label in SKILL_ROWS + ) + + f". {best[1]} is the best at the longest horizon and {worst[1]} the worst. " + f"Against this harder benchmark only {word(n_win_1)} of {word(len(SKILL_ROWS))} " + f"variables beat naive at one quarter and {word(n_win_last)} at {last}." + ) + + out = svg_open(W, H, "svar-skill-all", + "boe-svar forecast skill against a random walk with drift, " + "all eight variables", desc) + out.append('RMSE ÷ drifting-random-walk RMSE ' + '· horizons 1–8 quarters') + out.append('filled = difference significant at 5% ' + '(Diebold–Mariano); hollow = not distinguishable') + + for tick in (0.8, 0.9, 1.1, 1.2, 1.3): + x = xs(tick) + out.append(f'') + out.append(f'{tick}') + out.append(f'') + out.append(f'1.0') + out.append(f'' + f'← model better') + out.append(f'benchmark better →') + + for i, (key, label) in enumerate(SKILL_ROWS): + y = 78.0 + 38 * i + values = [ratio[h][key] for h in horizons] + out.append(f'' + f'{esc(label)}') + out.append(f'') + for h in horizons: + v, p = ratio[h][key], pval[h][key] + r = 4.0 if h == last else 2.6 + if p < SIGNIF: + series = 1 if v < 1 else 2 + out.append(f'') + else: + out.append(f'') + out.append(f'' + f'{ratio[last][key]:.2f}') + + out.append("") + return "\n".join(out) + + +def chart_svar_winrate(): + horizons, ratio, pval, _meta = load_rolling_eval() + keys = [k for k, _ in SKILL_ROWS] + total = len(keys) + + counts = [] + for h in horizons: + wins = [k for k in keys if ratio[h][k] < 1] + sig = [k for k in wins if pval[h][k] < SIGNIF] + counts.append({"h": h, "wins": len(wins), "sig": len(sig)}) + + W, H = 760, 300 + x0, x1 = 64, 724 + y_zero, y_top = 250.0, 60.0 + per_var = (y_zero - y_top) / total + + first_dry = next((c["h"] for c in counts if c["sig"] == 0), None) + desc = ( + f"Stacked column chart. Of the model's {word(total)} forecast variables, how many have " + f"lower root mean squared error than a random walk with drift, at horizons one to " + f"{word(horizons[-1])} quarters. The counts are " + + ", ".join(str(c["wins"]) for c in counts) + + " respectively. Each column also separates wins whose difference is statistically " + "significant at 5 per cent by a Diebold-Mariano test from wins that are not: " + "significant wins number " + + ", ".join(str(c["sig"]) for c in counts) + + "." + + (f" From horizon {first_dry} onward no variable beats the benchmark by a " + f"statistically significant margin." if first_dry else "") + ) + + out = svg_open(W, H, "svar-winrate", + "How many of eight variables beat a drifting random walk, by horizon", desc) + out.append(f'Of {total} forecast variables, how many ' + f'beat a random walk with drift') + + for k in (2, 4, 6, 8): + y = y_zero - k * per_var + out.append(f'') + out.append(f'{k}') + out.append(f'') + out.append(f'0') + + bar_w, pitch = 56, 80 + for i, c in enumerate(counts): + bx = x0 + 12 + i * pitch + mid = bx + bar_w / 2 + # Bottom-up: significant wins, then the rest of the wins, then the losses. + y_sig = y_zero - c["sig"] * per_var + y_win = y_zero - c["wins"] * per_var + if c["sig"]: + out.append(f'') + if c["wins"] > c["sig"]: + out.append(f'') + out.append(f'') + out.append(f'{c["wins"]}') + out.append(f'' + f'h={c["h"]}') + + out.append('' + 'beats it, and the difference is ' + 'significant') + out.append('' + 'beats it, not significantly') + out.append('' + 'does not beat it') + out.append("") + return "\n".join(out) + + +# Order must match the order the SVGs appear in validation/index.html: the +# substitution below is positional. BUILDERS = [ ("obr-anchored", chart_obr_anchored), ("obr-reform", chart_obr_reform), + ("obr-computed-share", chart_obr_computed_share), ("obr-freerun", chart_obr_freerun), ("obr-outturn", chart_obr_outturn), ("svar-fevd", chart_svar_fevd), ("svar-fan", chart_svar_fan), ("frbus-residuals", chart_frbus_residuals), + ("svar-skill-all", chart_svar_skill_all), + ("svar-winrate", chart_svar_winrate), ] SVG_RE = re.compile(r'', re.DOTALL) diff --git a/validation/index.html b/validation/index.html index e0e7879..fec0f43 100644 --- a/validation/index.html +++ b/validation/index.html @@ -299,28 +299,28 @@

Against the OBR's own forecast, and HMRC's reckoner.

- - How much of the OBR emulator scorecard the model actually computes - Two stacked bars. Of 21 headline variables in the OBR emulator calibration scorecard, 11 are actually computed by the model and 10 are passthrough, held at the OBR published value and therefore scoring zero error trivially. Of the 11 computed, 3 are fair (real GDP 5.75 per cent, consumption 9.56 per cent, trade balance 0.70 per cent of GDP), 1 is a trivial accounting identity (employment, where both inputs are themselves passthrough), 5 are poor and 2 are off (company profits 79.80 per cent, current account 4.17 per cent of GDP). Four of the 11, or 36 per cent, land within band, and one of those four is the trivial identity, so only three non-trivial computed variables are in band. - 21 headline scorecard variables - - - 11 computed - 10 passthrough — held at the OBR value - - - of which, the 11 the model computes - - fair 3 - - identity 1 - - poor 5 - - off 2 - Only 4 of the 11 computed variables land within band — 36% — and one of those four is a trivial identity. - bands: rates ±1.0pp · net balances ±1.5% of GDP · levels ≤10% MAPE - + +How much of the OBR emulator scorecard the model actually computes +Two stacked bars. Of 21 headline variables in the OBR emulator calibration scorecard, 11 are actually computed by the model and 10 are passthrough, held at the OBR published value and therefore scoring zero error trivially. Of the 11 computed, 3 are fair, 1 are identity, 5 are poor, 2 are off. 4 of the 11, or 36 per cent, land within band, and one of those is a trivial accounting identity, so only 3 non-trivial computed variables are in band. The worst are company profits 79.80 per cent and the current account 4.17 per cent of GDP. +21 headline scorecard variables + + +11 computed +10 passthrough — held at the OBR value + + +of which, the 11 the model computes + +fair 3 + +identity 1 + +poor 5 + +off 2 +Only 4 of the 11 computed variables land within band — 36% — and one of those four is a trivial identity. +bands: rates ±1.0pp · net balances ±1.5% of GDP · levels ≤10% MAPE +
Raw (un-anchored) calibration against the March 2026 EFO, a single vintage. The two “off” verdicts are bounded by constants the OBR does not publish — the OSHH base constant, the RPI index history, and the overseas rate-of-return normalisers — rather than by live bugs, and are gated against regression rather than tuned to the target. Source: docs/calibration_scorecard.md in the obr-macroeconomic-model repository.

@@ -532,7 +532,7 @@

Against a Bank of England paper, and against outturns.

boe-svar: global-shock FEVD shares, ours vs Brignone & Piffer (2025) -Grouped bar chart. Share of UK forecast-error variance attributed to identified global shocks (world demand, energy and supply) at the one-year horizon. For GDP, our 10,000-draw production run gives 42.1% against the paper's 40.0%; for CPI, 49.5% against 50.0%. GDP sits 2.1 points above the paper's figure, CPI half a point below. The paper's values are approximate. +Grouped bar chart. Share of UK forecast-error variance attributed to identified global shocks (world demand, energy and supply) at the one-year horizon. For GDP, our 10,000-draw production run gives 42.1% against the paper's 40.0%; for CPI, 49.5% against 50.0%. Both deviations are a percentage point or less. The paper's values are approximate. 0% @@ -1005,171 +1005,171 @@

boe-svar against a random walk, computed from our own runs

- boe-svar forecast skill against a random walk with drift, all eight variables - Dot matrix of forecast error relative to a random walk with drift, eight variables by eight quarterly horizons, from 49 expanding-window origins spanning 2012Q1 to 2024Q1. A ratio below 1.0 means the model beats the benchmark; filled dots mark differences significant at 5 per cent by a Diebold-Mariano test, hollow dots differences that are not statistically distinguishable. Bank Rate is the only variable that beats the benchmark at every horizon, ranging 0.79 to 0.89, and its one-quarter advantage is significant (p = 0.018). UK CPI starts at 0.83 at one quarter, significant at p = 0.031, but drifts to 1.03 by eight quarters, losing its advantage. World CPI follows the same pattern, 0.94 rising to 1.07. Oil price and CPI energy sit near parity throughout. UK real GDP (1.06 to 1.12) and world GDP (1.07 to 1.10) stay just above parity, but none of those differences is statistically significant. The exchange rate is worst and worsens monotonically from 1.03 to 1.33. Against this harder benchmark only 4 of eight variables beat naive at one quarter and 2 at eight. - RMSE ÷ drifting-random-walk RMSE · horizons 1–8 quarters - filled = difference significant at 5% (Diebold–Mariano); hollow = not distinguishable - - 0.8 - - 0.9 - - 1.1 - - 1.2 - - 1.3 - - 1.0 - ← model better - benchmark better → - Bank Rate - - - - - - - - - - 0.85 - UK CPI - - - - - - - - - - 1.03 - World CPI - - - - - - - - - - 1.07 - Oil price - - - - - - - - - - 0.98 - CPI energy - - - - - - - - - - 1.11 - UK real GDP - - - - - - - - - - 1.12 - World GDP - - - - - - - - - - 1.10 - Exchange rate - - - - - - - - - - 1.33 - +boe-svar forecast skill against a random walk with drift, all eight variables +Dot matrix of forecast error relative to a random walk with drift, eight variables by eight quarterly horizons, from 49 expanding-window origins. A ratio below 1.0 means the model beats the benchmark; filled dots mark differences significant at 5 per cent by a Diebold-Mariano test, hollow dots differences that are not statistically distinguishable. Bank Rate runs 0.79 at one quarter to 0.85 at 8; UK CPI runs 0.83 at one quarter to 1.03 at 8; World CPI runs 0.94 at one quarter to 1.07 at 8; Oil price runs 1.01 at one quarter to 0.98 at 8; CPI energy runs 0.98 at one quarter to 1.11 at 8; UK real GDP runs 1.06 at one quarter to 1.12 at 8; World GDP runs 1.07 at one quarter to 1.10 at 8; Exchange rate runs 1.03 at one quarter to 1.33 at 8. Bank Rate is the best at the longest horizon and Exchange rate the worst. Against this harder benchmark only four of eight variables beat naive at one quarter and two at 8. +RMSE ÷ drifting-random-walk RMSE · horizons 1–8 quarters +filled = difference significant at 5% (Diebold–Mariano); hollow = not distinguishable + +0.8 + +0.9 + +1.1 + +1.2 + +1.3 + +1.0 +← model better +benchmark better → +Bank Rate + + + + + + + + + +0.85 +UK CPI + + + + + + + + + +1.03 +World CPI + + + + + + + + + +1.07 +Oil price + + + + + + + + + +0.98 +CPI energy + + + + + + + + + +1.11 +UK real GDP + + + + + + + + + +1.12 +World GDP + + + + + + + + + +1.10 +Exchange rate + + + + + + + + + +1.33 +
Expanding-window pseudo-out-of-sample error ratios against a random walk with drift, on the level of each series. A driftless walk is too weak a benchmark for a trending series, so this is the fair comparison; the no-change ratios are in the table below. 49 forecast origins, 2012Q1–2024Q1 (the window includes the Covid quarters); estimation sample from 1992Q1, data through 2026Q1; 4 lags. Estimation uses final revised data, so this is pseudo- rather than real-time out-of-sample. No single origin contributes more than 35% of any drift-benchmark squared error. Source: papers/boe-svar/figures/rolling_evaluation.json.
- How many of eight variables beat a drifting random walk, by horizon - Stacked column chart. Of the model's eight forecast variables, how many have lower root mean squared error than a random walk with drift, at horizons one to eight quarters. The counts are 4, 4, 3, 2, 3, 3, 2, 2 respectively. Each column also separates wins whose difference is statistically significant at 5 per cent by a Diebold-Mariano test from wins that are not: significant wins number 2, 1, 1, 1, 1, 0, 0, 0. From horizon six onward no variable beats the benchmark by a statistically significant margin. At one quarter the two significant wins are Bank Rate and UK CPI; beyond four quarters only Bank Rate remains ahead at all. - Of 8 forecast variables, how many beat a random walk with drift - - 2 - - 4 - - 6 - - 8 - - 0 - - - - 4 - h=1 - - - - 4 - h=2 - - - - 3 - h=3 - - - - 2 - h=4 - - - - 3 - h=5 - - - 3 - h=6 - - - 2 - h=7 - - - 2 - h=8 - beats it, and the difference is significant - beats it, not significantly - does not beat it - +How many of eight variables beat a drifting random walk, by horizon +Stacked column chart. Of the model's eight forecast variables, how many have lower root mean squared error than a random walk with drift, at horizons one to eight quarters. The counts are 4, 4, 3, 2, 3, 3, 2, 2 respectively. Each column also separates wins whose difference is statistically significant at 5 per cent by a Diebold-Mariano test from wins that are not: significant wins number 2, 1, 1, 1, 1, 0, 0, 0. From horizon 6 onward no variable beats the benchmark by a statistically significant margin. +Of 8 forecast variables, how many beat a random walk with drift + +2 + +4 + +6 + +8 + +0 + + + +4 +h=1 + + + +4 +h=2 + + + +3 +h=3 + + + +2 +h=4 + + + +3 +h=5 + + +3 +h=6 + + +2 +h=7 + + +2 +h=8 +beats it, and the difference is significant +beats it, not significantly +does not beat it +
Wins are counted at a ratio strictly below 1.0, which is a hard cut: a variable at 0.99 counts as a win and one at 1.01 does not, even though neither is distinguishable from the benchmark. That is why the significance split matters more than the count. Same 49 origins and benchmark as the chart above. Source: papers/boe-svar/figures/rolling_evaluation.json.