library(dplyr) # data manipulationlibrary(modelsummary) # regression tableslibrary(marginaleffects) # predictions and marginal effectslibrary(suest) # seemingly unrelated estimation# Create some data for rescaling example (logit vs OLS)# (R's random numbers differ from Stata's, so the results differ a little)set.seed(30004)n <-10000sim <-data.frame(x1 =as.numeric(1:n > n /2), # first half = 0, second half = 1x2 = (1:n) %%2) # alternates 1/0 within each halfsim$yB <-as.numeric(runif(n) <plogis(-1+2.5* sim$x1 -2.5* sim$x2))sim$yC <-0+2.5* sim$x1 -2.5* sim$x2 +rnorm(n, 0, 3.5)round(sapply(sim[c("yB", "yC", "x1", "x2")],function(v) c(mean =mean(v), sd =sd(v), min =min(v), max =max(v))), 2)
yB yC x1 x2
mean 0.35 -0.03 0.5 0.5
sd 0.48 3.96 0.5 0.5
min 0.00 -15.40 0.0 0.0
max 1.00 13.83 1.0 1.0
The effect size in logit is 2 times larger once we account for an orthogonal IV!?!?
Intuitive explanation of rescaling
“The variance of Y* changes because estimation of the probability of success gets better and better as you add more variables, and better estimation in turn leads to more variability in the estimates. Suppose you had a single dichotomous independent variable… (gender)… All men would have the same predicted probability of success and all women would also share the same predicted probability. For example, men might have a 40% probability of success while women have a 50% chance.
As you add more variables, however, there will start to be more variability in the individuals’ predicted probabilities. So, among the men, after adding another variable the probabilities might now range between 35% and 45%, while for women the range might be 45%–55%. Hence, instead of the probabilities only ranging from 40% to 50%, adding another variable could cause the individual probabilities to range from 35% to 55%. If you are doing really well, adding more variables might even get you to the point where the predicted individual probabilities of success ranged from 0.0001 to 0.9999.”
Consider the following where \(b_{yx}\) is the observed coefficient for focal IV x in model R (reduced model without mediator) and \(b_{yx \bullet z}\) is the observed coefficient for x in model F (full model with mediator z included)
\(\beta_{yx}\) and \(\beta_{yx \bullet z}\) are the true effects
The difference \(b_{yx} - b_{yx \bullet z}\) does not equal \(\beta_{yx} - \beta_{yx \bullet z}\) because the scale parameters (σ) are not equal across models
If the additional variables in model F explain any variation in the outcome then \(\sigma_{R} > \sigma_{F}\) and thus \(b_{yx} - b_{yx \bullet z} \neq \beta_{yx} - \beta_{yx \bullet z}\)
The exact details of the KHB method are beyond the scope of this course—the key takeaway is that their solution allows the coefficients to be on the same scale so that they can be directly compared: significance tests from this method are appropriate tests of cross-model differences
Though the coefficient in the reduced model no longer has its natural interpretation
Because the coefficient from the reduced model is no longer directly interpretable, another way to represent the cross-model difference is the ratio of the two same-scale coefficients across models
Which can easily be represented as the proportion change in the size of the coefficient across the two models:
\[1 - \frac{\beta_{yx \bullet z}}{\beta_{yx}}\]
Add Health example: mediation in logit pr(depress)
Example: Add Health Wave 4
DV: Depressive = 1
Binary version of depressive symptoms scale representing at risk
library(haven) # import Stata data and labelsah4 <-read_dta("https://tdmize.github.io/data/data/addhealth4.dta")# KHB examplereduced <-glm(depB ~ edyrs, data = ah4, family =binomial(link ="logit"))full <-glm(depB ~ edyrs + wages, data = ah4, family =binomial(link ="logit"))modelsummary(list(reduced, full), fmt =3,stars =c("*"= .05, "**"= .01, "***"= .001))
(1)
(2)
* p < 0.05, ** p < 0.01, *** p < 0.001
(Intercept)
0.814***
0.736**
(0.242)
(0.242)
edyrs
-0.141***
-0.119***
(0.017)
(0.018)
wages
-0.015***
(0.003)
Num.Obs.
4307
4307
AIC
4587.4
4562.1
BIC
4600.1
4581.2
Log.Lik.
-2291.695
-2278.071
F
69.428
43.607
RMSE
0.42
0.42
The problem with eyeballing these effect sizes is we don’t know how much of the change in the coefficient on education (-0.141 to -0.119) is due to including wages in the model, and how much is due to model rescaling
The khb method to test for mediation
khb command in Stata; in R, calculate it by hand (see the R code)
Decomposition using the KHB-Method
Model-Type: logit Number of obs = 4307
Variables of Interest: edyrs Pseudo R2 = 0.02
Z-variable(s): wages
------------------------------------------------------------------------------
depB | Coefficient Std. err. z P>|z| [95% conf. interval]
-------------+----------------------------------------------------------------
edyrs |
Reduced | -0.146 0.017 -8.580 0.000 -0.180 -0.113
Full | -0.119 0.018 -6.781 0.000 -0.153 -0.084
Diff | -0.028 0.006 -4.252 0.000 -0.040 -0.015
------------------------------------------------------------------------------
Summary of confounding
Variable | Conf_ratio Conf_Pct Resc_Fact
-------------+-------------------------------------
edyrs | 1.2319681 18.83 1.036583
---------------------------------------------------
# R has no working built-in way to do the KHB method for logit models (the khb# function in the matchingMarkets package fits probit models only), so calculate# it by hand. The reduced model uses the part of the mediator (z) unrelated to the# focal IV (x): its residual from a regression on x. Both models are then on the# same scale, and the difference is the indirect effect.khb_logit <-function(y, x, z, data) { aux <-lm(reformulate(x, z), data = data) # z regressed on x data$z_res <-resid(aux) naive <-glm(reformulate(x, y), data = data, family =binomial(link ="logit")) reduced <-glm(reformulate(c(x, "z_res"), y), data = data, family =binomial(link ="logit")) full <-glm(reformulate(c(x, z), y), data = data, family =binomial(link ="logit")) b_red <-coef(reduced)[[x]] b_full <-coef(full)[[x]] est <-c(Reduced = b_red, Full = b_full, Diff = b_red - b_full) se <-c(sqrt(vcov(reduced)[x, x]), sqrt(vcov(full)[x, x]),# Diff = (coef of z in full model) * (coef of x on z); delta methodsqrt(coef(aux)[[x]]^2*vcov(full)[z, z] +coef(full)[[z]]^2*vcov(aux)[x, x])) zval <- est / seprint(round(cbind(Coef = est, "Std. Err."= se, z = zval, "P>|z|"=2*pnorm(-abs(zval)),"[95% Conf."= est -qnorm(.975) * se, "Interval]"= est +qnorm(.975) * se), 3))cat("\nSummary of confounding\n")round(c(Conf_ratio = b_red / b_full, Conf_Pct =100* (b_red - b_full) / b_red,Resc_Fact = b_red /coef(naive)[[x]]), 4)}khb_logit("depB", "edyrs", "wages", data = ah4)
The effect of education is reduced after accounting for wages (p < 0.001). About 19% of the reduction in the coefficient is due to the inclusion of wages in the model
khb method applied to our rescaling example
Recall the example with data I generated
We know that the true effect of x1 and x2 should not change across models as x1 and x2 are orthogonal
But, the odds ratios from logit changed dramatically
*Simulated data from the top ofthis page (see the code there)khb logit yB x1 || x2, summary
Decomposition using the KHB-Method
Model-Type: logit Number of obs = 10000
Variables of Interest: x1 Pseudo R2 = 0.31
Z-variable(s): x2
------------------------------------------------------------------------------
yB | Coefficient Std. err. z P>|z| [95% conf. interval]
-------------+----------------------------------------------------------------
x1 |
Reduced | 2.455 0.060 41.007 0.000 2.337 2.572
Full | 2.455 0.060 41.007 0.000 2.337 2.572
Diff | 0.000 0.025 0.000 1.000 -0.049 0.049
------------------------------------------------------------------------------
Summary of confounding
Variable | Conf_ratio Conf_Pct Resc_Fact
-------------+-------------------------------------
x1 | 1 0.00 1.304165
---------------------------------------------------
# KHB method with the simulated data (x1 and x2 as 0/1 numbers)simn <-data.frame(yB = sim$yB, x1 =as.numeric(sim$x1 =="1"),x2 =as.numeric(sim$x2 =="1"))khb_logit("yB", "x1", "x2", data = simn)
*Compare the MEs across modelsmecompare x1, models(logit_x1 logit_x1_x2)
mecompare uses vce(robust) for all models. Standard errors from mecompare will d
> iffer from the specified model(s) because vce(robust) was not used on at least
> one of the models specified in the models( ) option. We strongly recommend re
> fitting the first (logit) and second (logit) models with vce(robust). See vce_
> option for details on vce(robust).
Predicting: Pr(yB)
Marginal effects and cross-model differences (N_logit_x1=10000)
(N_logit_x1_x2=10000)
| ME # Estimate Robust SE P>|z|
---------------------------------+---------------------------------------
x1 |
1 - 0 |
logit x1 | 1 0.390 0.009 0.000
logit x1 x2 | 2 0.390 0.008 0.000
Difference | 3 0.000 0.004 1.000
mecompare x2, models(logit_x2 logit_x1_x2)
mecompare uses vce(robust) for all models. Standard errors from mecompare will d
> iffer from the specified model(s) because vce(robust) was not used on at least
> one of the models specified in the models( ) option. We strongly recommend re
> fitting the first (logit) and second (logit) models with vce(robust). See vce_
> option for details on vce(robust).
Predicting: Pr(yB)
Marginal effects and cross-model differences (N_logit_x2=10000)
(N_logit_x1_x2=10000)
| ME # Estimate Robust SE P>|z|
---------------------------------+---------------------------------------
x2 |
1 - 0 |
logit x2 | 1 -0.395 0.009 0.000
logit x1 x2 | 2 -0.395 0.008 0.000
Difference | 3 -0.000 0.004 1.000
# Compare the MEs across modelsfit_x1 <-suest(logit_x1, logit_x1_x2, model_names =c("logit x1", "logit x1 x2"))me_x1 <-avg_comparisons(fit_x1, variables ="x1", newdata = sim)me_x1