Skip to content

Commit 469a829

Browse files
committed
Add Stata did_imputation benchmark harness
Generates identical synthetic panels for Julia and Stata, times our BJS estimator against Stata SE's did_imputation on a matched spec. Generated data/results are gitignored.
1 parent e2c4e78 commit 469a829

3 files changed

Lines changed: 148 additions & 0 deletions

File tree

bench/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Generated benchmark artifacts — regenerate with run_bench_julia.jl / bench_stata.do
2+
data_*.csv
3+
results_*.csv
4+
results_*.dta
5+
*.log

bench/bench_stata.do

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
* Benchmark Stata `did_imputation` (Borusyak et al.) on the SAME CSV panels that
2+
* run_bench_julia.jl generated. Matched spec: all post horizons, 4 pre-trends,
3+
* clustered on unit, minn(0), tol(1e-6), maxit(100), autosample.
4+
*
5+
* Stata SE is single-threaded, so this is the like-for-like comparison against
6+
* our single-threaded estimator. Run from the bench/ directory.
7+
8+
clear all
9+
set more off
10+
version 17
11+
12+
* did_imputation needs reghdfe + ftools; confirm they're present.
13+
capture which reghdfe
14+
if _rc {
15+
di as error "reghdfe not installed -- run: ssc install reghdfe, ftools"
16+
exit 198
17+
}
18+
19+
local cases S M L
20+
21+
* Warm up the ado / Mata libraries on a tiny run so they aren't timed below.
22+
quietly {
23+
import delimited "data_S.csv", clear case(preserve)
24+
capture did_imputation Y id t Ei, allhorizons pretrends(4) cluster(id) ///
25+
minn(0) tol(0.000001) maxit(100) autosample
26+
}
27+
28+
* Results accumulator.
29+
postutil clear
30+
tempname pf
31+
postfile `pf' str4 case long nobs int ncoef double stata_s double b0 ///
32+
using "results_stata.dta", replace
33+
34+
foreach c of local cases {
35+
import delimited "data_`c'.csv", clear case(preserve)
36+
local nobs = _N
37+
38+
* Two timed runs, keep the faster (reduces noise; no JIT so both comparable).
39+
local best = .
40+
forvalues r = 1/2 {
41+
timer clear 1
42+
timer on 1
43+
did_imputation Y id t Ei, allhorizons pretrends(4) cluster(id) ///
44+
minn(0) tol(0.000001) maxit(100) autosample
45+
timer off 1
46+
quietly timer list 1
47+
if (`best' == . | r(t1) < `best') local best = r(t1)
48+
}
49+
50+
local ncoef = colsof(e(b))
51+
* Horizon-0 effect for cross-check against Julia's b(τ0).
52+
local b0 = .
53+
capture local b0 = _b[tau0]
54+
55+
post `pf' ("`c'") (`nobs') (`ncoef') (`best') (`b0')
56+
di as txt "case `c': nobs=`nobs' ncoef=`ncoef' time=`best's b(tau0)=`b0'"
57+
}
58+
59+
postclose `pf'
60+
use "results_stata.dta", clear
61+
export delimited using "results_stata.csv", replace
62+
list, clean noobs

bench/run_bench_julia.jl

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Benchmark: StagDiDModels BJS imputation estimator vs Stata `did_imputation`.
2+
#
3+
# This script (1) generates synthetic staggered-adoption panels, writing each to
4+
# a CSV that BOTH Julia and Stata read (identical data → fair comparison), and
5+
# (2) times our `fit_bjs` estimator on each, for both the optimized default path
6+
# (`multithreaded=true`) and the naive path (`multithreaded=false`). Warm timings
7+
# only (compilation excluded) — Stata's interpreted ado has no JIT, so steady
8+
# per-call runtime is the like-for-like quantity.
9+
#
10+
# Matched spec (see bench_stata.do): dynamic event study, all post horizons,
11+
# 4 pre-trends, clustered on unit, minn=0, tol=1e-6, maxit=100, autosample.
12+
13+
using StagDiDModels, DataFrames, CSV, Random, Printf, Statistics
14+
15+
const BENCHDIR = @__DIR__
16+
const PRETRENDS = 4
17+
const TOL = 1e-6
18+
const MAXIT = 100 # match Stata did_imputation default maxit(100)
19+
20+
"Deterministic staggered panel: ~30% never-treated, rest in cohorts {5,7,9}."
21+
function gen_panel(N::Int, T::Int; seed::Int=1)
22+
rng = MersenneTwister(seed)
23+
cohorts = (5, 7, 9)
24+
α = randn(rng, N) # unit effects
25+
γ = randn(rng, T) # time effects
26+
ids = Int[]; ts = Int[]
27+
Ei = Vector{Union{Int,Missing}}(); Y = Float64[]
28+
sizehint!(ids, N*T); sizehint!(ts, N*T); sizehint!(Ei, N*T); sizehint!(Y, N*T)
29+
for u in 1:N
30+
ei = (u % 10 < 3) ? missing : cohorts[(u % 3) + 1]
31+
for tt in 1:T
32+
treated = !ismissing(ei) && tt >= ei
33+
k = ismissing(ei) ? 0 : tt - ei
34+
y = α[u] + γ[tt] + (treated ? 0.3 * (k + 1) : 0.0) + 0.5 * randn(rng)
35+
push!(ids, u); push!(ts, tt); push!(Ei, ei); push!(Y, y)
36+
end
37+
end
38+
DataFrame(id = ids, t = ts, Ei = Ei, Y = Y)
39+
end
40+
41+
fit(df; mt) = fit_bjs_dynamic(df; y=:Y, id=:id, t=:t, g=:Ei,
42+
horizons=true, pretrends=PRETRENDS, cluster=:id,
43+
minn=0, tol=TOL, maxiter=MAXIT, autosample=true,
44+
multithreaded=mt)
45+
46+
"Warm up once, then return minimum elapsed over `reps` runs."
47+
function timeit(f; reps::Int=3)
48+
f() # warmup / compile for this data shape
49+
best = Inf
50+
for _ in 1:reps
51+
GC.gc()
52+
best = min(best, @elapsed f())
53+
end
54+
best
55+
end
56+
57+
coefat(m, name) = (i = findfirst(==(name), coefnames(m)); i === nothing ? NaN : coef(m)[i])
58+
59+
# (label, N, T)
60+
const CASES = [("S", 1_000, 12), ("M", 4_000, 12), ("L", 10_000, 12)]
61+
62+
rows = NamedTuple[]
63+
@printf("%-4s %8s %6s %6s %12s %12s %10s %10s\n",
64+
"case", "nobs", "ncoef", "iters", "opt_s", "naive_s", "speedup", "b(τ0)")
65+
for (label, N, T) in CASES
66+
df = gen_panel(N, T)
67+
CSV.write(joinpath(BENCHDIR, "data_$(label).csv"), df)
68+
69+
m = fit(df; mt=true)
70+
ncoef = length(coef(m)); b0 = coefat(m, "τ::0")
71+
opt = timeit(() -> fit(df; mt=true))
72+
naive = timeit(() -> fit(df; mt=false))
73+
74+
push!(rows, (case=label, nobs=nrow(df), ncoef=ncoef,
75+
opt_s=opt, naive_s=naive, b0=b0))
76+
@printf("%-4s %8d %6d %6s %12.4f %12.4f %9.2fx %10.4f\n",
77+
label, nrow(df), ncoef, "-", opt, naive, naive/opt, b0)
78+
end
79+
80+
CSV.write(joinpath(BENCHDIR, "results_julia.csv"), DataFrame(rows))
81+
println("\nWrote data_*.csv and results_julia.csv to $BENCHDIR")

0 commit comments

Comments
 (0)