Merge branch 'revision-2' of https://github.com/paxtonfitzpatrick/efficient-learning-khan

Author: jeremymanning

paper/CDL-bibliography/cdl.bib

@@ -1,7 +1,80 @@
 
 
+@article{NortEtal02,
+	author = {North, B V and Curtis, D and Sham, P C},
+	date-added = {2024-02-21 18:40:44 -0500},
+	date-modified = {2024-02-21 18:43:17 -0500},
+	doi = {10.1086/341527},
+	journal = {American Journal of Human Genetics},
+	month = {Aug},
+	number = {2},
+	pages = {439--441},
+	title = {A note on the calculation of empirical P values from Monte Carlo procedures.},
+	volume = {71},
+	year = {2002}}
+
+@book{DaviHink97,
+	author = {Davison, A. C. and Hinkley, D. V.},
+	date-modified = {2024-02-21 18:39:11 -0500},
+	month = {October},
+	publisher = {Cambridge University Press},
+	series = {Cambridge Series in Statistical and Probabilistic Mathematics},
+	title = {Bootstrap Methods and their Application},
+	year = {1997}}
+
+@incollection{SnijBosk11,
+	author = {T A B Snijders and R Bosker},
+	booktitle = {{Multilevel Analysis: An Introduction to Basic and Advanced Multilevel Modeling}},
+	chapter = {6},
+	date-added = {2024-02-21 18:08:36 -0500},
+	date-modified = {2024-02-21 18:29:12 -0500},
+	edition = {2nd},
+	month = {November},
+	pages = {94--108},
+	publisher = {Sage Publications},
+	title = {More powerful tests for variance parameters},
+	year = {2011}}
+
+@article{ScheEtal08b,
+	author = {Fabian Scheipl and Sonja Greven and Helmut K{\"u}chenhoff},
+	date-added = {2024-02-21 17:29:35 -0500},
+	date-modified = {2024-02-21 17:30:19 -0500},
+	doi = {https://doi.org/10.1016/j.csda.2007.10.022},
+	issn = {0167--9473},
+	journal = {Computational Statistics \& Data Analysis},
+	number = {7},
+	pages = {3283-3299},
+	title = {Size and power of tests for a zero random effect variance or polynomial regression in additive and linear mixed models},
+	volume = {52},
+	year = {2008}}
+
+@article{GoldSimo00,
+	author = {Nick Goldman and Simon Whelan},
+	date-added = {2024-02-21 17:18:14 -0500},
+	date-modified = {2024-02-21 17:19:25 -0500},
+	doi = {10.1093/oxfordjournals.molbev.a026378},
+	issn = {0737--4038},
+	journal = {Molecular Biology and Evolution},
+	month = {06},
+	number = {6},
+	pages = {975-978},
+	title = {{Statistical Tests of Gamma-Distributed Rate Heterogeneity in Models of Sequence Evolution in Phylogenetics}},
+	volume = {17},
+	year = {2000}}
+
+@article{HaleHojs14,
+	author = {Halekoh, Ulrich and H{\o}jsgaard, S{\o}ren},
+	date-added = {2024-02-21 17:05:30 -0500},
+	date-modified = {2024-02-21 17:05:45 -0500},
+	doi = {10.18637/jss.v059.i09},
+	journal = {Journal of Statistical Software},
+	number = {9},
+	pages = {1--32},
+	title = {{A Kenward-Roger Approximation and Parametric Bootstrap Methods for Tests in Linear Mixed Models -- The R Package pbkrtest}},
+	volume = {59},
+	year = {2014}}
+
 @article{BarrEtal13,
-	abstract = {Linear mixed-effects models (LMEMs) have become increasingly prominent in psycholinguistics and related areas. However, many researchers do not seem to appreciate how random effects structures affect the generalizability of an analysis. Here, we argue that researchers using LMEMs for confirmatory hypothesis testing should minimally adhere to the standards that have been in place for many decades. Through theoretical arguments and Monte Carlo simulation, we show that LMEMs generalize best when they include the maximal random effects structure justified by the design. The generalization performance of LMEMs including data-driven random effects structures strongly depends upon modeling criteria and sample size, yielding reasonable results on moderately-sized samples when conservative criteria are used, but with little or no power advantage over maximal models. Finally, random-intercepts-only LMEMs used on within-subjects and/or within-items data from populations where subjects and/or items vary in their sensitivity to experimental manipulations always generalize worse than separate F1 and F2 tests, and in many cases, even worse than F1 alone. Maximal LMEMs should be the `gold standard' for confirmatory hypothesis testing in psycholinguistics and beyond.},
 	author = {Dale J. Barr and Roger Levy and Christoph Scheepers and Harry J. Tily},
 	date-added = {2024-02-20 10:18:31 -0500},
 	date-modified = {2024-02-20 10:18:52 -0500},
@@ -25798,7 +25871,8 @@ @article{GoloTaub99
 	publisher = {Society for Neuroscience},
 	title = {Head direction cells in rats with hippocampal or overlying neocortical lesions: evidence for impaired angular path integration},
 	volume = {19},
-	year = {1999}}
+	year = {1999},
+	bdsk-file-1 = {YnBsaXN0MDDSAQIDBFxyZWxhdGl2ZVBhdGhZYWxpYXNEYXRhXxA1Li4vLi4vLi4vLi4vLi4vLi4vRG93bmxvYWRzL1MwMTY3OTQ3MzA3MDA0MzA2ICgxKS5iaWJPEQF0AAAAAAF0AAIAAAxNYWNpbnRvc2ggSEQAAAAAAAAAAAAAAAAAAADhKyW+QkQAAf////8ZUzAxNjc5NDczMDcwMDQzMDYgKDEpLmJpYgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/////+H740kAAAAAAAAAAAAGAAIAAAogY3UAAAAAAAAAAAAAAAAACURvd25sb2FkcwAAAgAyLzpVc2VyczpwYXh0b246RG93bmxvYWRzOlMwMTY3OTQ3MzA3MDA0MzA2ICgxKS5iaWIADgA0ABkAUwAwADEANgA3ADkANAA3ADMAMAA3ADAAMAA0ADMAMAA2ACAAKAAxACkALgBiAGkAYgAPABoADABNAGEAYwBpAG4AdABvAHMAaAAgAEgARAASADBVc2Vycy9wYXh0b24vRG93bmxvYWRzL1MwMTY3OTQ3MzA3MDA0MzA2ICgxKS5iaWIAEwABLwAAFQACAA3//wAAAAgADQAaACQAXAAAAAAAAAIBAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAHU}}
 
 @article{RobeEtal99,
 	author = {R G Robertson and E T Rolls and P Georges-Fran{\c{c}}ois and S Panzeri},
@@ -31915,8 +31989,9 @@ @article{Srin99
 	volume = {1},
 	year = {1999}}
 
-@article{ScheEtal08,
+@article{ScheEtal08a,
 	author = {C A Schevon and S K Ng and J Cappell and R R Goodman and G McKhann and A Waziri and A Branner and A Sosunov and C E Schroeder and R G Emerson},
+	date-modified = {2024-02-21 17:30:06 -0500},
 	journal = {Journal of Clinical Neurophysiology},
 	number = {6},
 	pages = {321--330},
@@ -35180,7 +35255,8 @@ @article{DawEtal02
 	pages = {603--616},
 	title = {Opponent interactions between serotonin and dopamine},
 	volume = {15},
-	year = {2002}}
+	year = {2002},
+	bdsk-file-1 = {YnBsaXN0MDDSAQIDBFxyZWxhdGl2ZVBhdGhZYWxpYXNEYXRhXxArLi4vLi4vLi4vLi4vLi4vLi4vRG93bmxvYWRzL3BtY18zNzkxNzgubmJpYk8RAUwAAAAAAUwAAgAADE1hY2ludG9zaCBIRAAAAAAAAAAAAAAAAAAAAOErJb5CRAAB/////w9wbWNfMzc5MTc4Lm5iaWIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD/////4fv0EgAAAAAAAAAAAAYAAgAACiBjdQAAAAAAAAAAAAAAAAAJRG93bmxvYWRzAAACACgvOlVzZXJzOnBheHRvbjpEb3dubG9hZHM6cG1jXzM3OTE3OC5uYmliAA4AIAAPAHAAbQBjAF8AMwA3ADkAMQA3ADgALgBuAGIAaQBiAA8AGgAMAE0AYQBjAGkAbgB0AG8AcwBoACAASABEABIAJlVzZXJzL3BheHRvbi9Eb3dubG9hZHMvcG1jXzM3OTE3OC5uYmliABMAAS8AABUAAgAN//8AAAAIAA0AGgAkAFIAAAAAAAACAQAAAAAAAAAFAAAAAAAAAAAAAAAAAAABog==}}
 
 @article{KirsEtal00,
 	author = {M P Kirschen and M J Kahana and R Sekuler and B Burack},

paper/changes.pdf

@@ -1,7 +1,7 @@
 \documentclass[10pt]{article}
 %DIF LATEXDIFF DIFFERENCE FILE
 %DIF DEL old.tex    Mon Feb 19 07:49:49 2024
-%DIF ADD main.tex   Wed Feb 21 12:35:29 2024
+%DIF ADD main.tex   Wed Feb 21 19:15:05 2024
 %DIF 2a2
 \usepackage{amsmath} %DIF > 
 %DIF -------
@@ -1745,13 +1745,11 @@ \subsubsection*{Estimating dynamic knowledge traces}\label{subsec:traces}
 Fundamental Forces}}\DIFadd{, }\textit{\DIFadd{Birth of Stars}}\DIFadd{, or general physics knowledge.
 Note that with our coding scheme, identifiers for each }\texttt{\DIFadd{question}} \DIFadd{are
 implicitly nested within levels of }\texttt{\DIFadd{lecture}} \DIFadd{and do not require explicit
-nesting in our model formula.
+nesting in our model formula. We then iteratively removed random effects from 
+the maximal model until it successfully converged with a full rank (i.e., non-singular) 
+random effects variance-covariance matrix.
 }
 
-%DIF >  We then iteratively removed random effects from the maximal model until it
-%DIF >  successfully converged with a full rank (i.e., non-singular) random effects
-%DIF >  variance-covariance matrix.
-
 %DIF > % JRM NOTE: do we need this next paragraph?  Commenting out for now...
 %DIF >  %When inspecting the model's random effect estimates revealed multiple terms estimated at the boundary of their parameter space (i.e., variance components of 0 or correlation terms of  $\pm 1$), we found that the order in which we eliminated these terms typically did not affect which terms did and did not need to be removed in order for the model to converge to a non-degenerate solution.
 %DIF >  When this required eliminating multiple terms whose estimates reached the boundary of their parameter space (i.e., variance components of 0 or correlation terms of  $\pm 1$), we found that the order in which we did so typically did not change the set of terms that needed to be removed in order for the model to converge to a non-degenerate solution.
@@ -1775,98 +1773,28 @@ \subsubsection*{Estimating dynamic knowledge traces}\label{subsec:traces}
 
 \DIFadd{To assess the predictive value of our knowledge estimates, we compared each
 GLMM's ability to discriminate between correctly and incorrectly answered
-questions to that of an analogous model that did not consider estimated
+questions to that of an analogous model that did }\textit{\DIFadd{not}} \DIFadd{consider estimated
 knowledge. Specifically, we used the same sets of observations with which we
 fit each ``full'' model to fit a second ``null'' model, with the formula:
 }\[
     \DIFadd{\mathtt{accuracy \sim (1\ \vert\ participant) + (1\ \vert\ question)}
 }\]
-\DIFadd{where the terms are as defined above.
+\DIFadd{where ``}\texttt{\DIFadd{accuracy}}\DIFadd{'', ``}\texttt{\DIFadd{participant}}\DIFadd{'', and ``}\texttt{\DIFadd{question}}\DIFadd{'' are as defined above. 
+As with our full models, the null models we fit for the “All questions” version of the analysis for each quiz contained an additional term, $\mathtt{(1\ \vert\ lecture)}$, where ``}\texttt{\DIFadd{lecture}}\DIFadd{'' are as defined above. 
+We then compared each full model to its reduced (null) equivalent using a likelihood-ratio test (LRT). 
+Because the typical asymptotic $\chi^2_d$ approximation of the null distribution for the LRT statistic ($\lambda_{LR}$) is anti-conservative for models that differ in their random slope terms~\mbox{%DIFAUXCMD
+\citep{GoldSimo00,ScheEtal08b,SnijBosk11}}\hskip0pt%DIFAUXCMD
+, we computed $p$-values for these tests using a parametric bootstrapping procedure~\mbox{%DIFAUXCMD
+\citep{HaleHojs14}}\hskip0pt%DIFAUXCMD
+. 
+For each of 1,000 bootstraps, we used the fitted null model to simulate a sample of observations of equal size to our original sample.
+We then re-fit both the null and full models to this simulated sample and compared them via an LRT.
+This yielded a distribution of $\lambda_{LR}$ statistics we may expect to observe under our null hypothesis.
+Following~\mbox{%DIFAUXCMD
+\citep{DaviHink97,NortEtal02}}\hskip0pt%DIFAUXCMD
+, we computed a corrected $p$-value for our observed $\lambda_{LR}$ as $\frac{r + 1}{n + 1}$, where $r$ is the number of simulated model comparisons that yielded a $\lambda_{LR}$ greater than or equal to our observed value and $n$ is the number of simulations we ran (1,000).
 }
 
-
-
-
-
-
-
-
-
-%DIF > In order to assess the predictive value of the knowledge estimates, we then fit a second set of ``null'' models to the same sets of observations used to fit our full GLMMs. These 
-%DIF > 
-%DIF > 
-%DIF > used the same sets of observations used to fit these GLMMs to fit a second set of ``null'' models. 
-%DIF > 
-%DIF > 
-%DIF > 
-%DIF > Next, in order to assess the predictive value of the knowledge estimates 
-%DIF > 
-%DIF > 
-%DIF > 
-%DIF > In order to assess the predictive value of the knowledge estimates we used to fit each GLMM, we then fit a \textit{second} model to the same data as each of the 15 
-%DIF > 
-%DIF > In order to assess whether the knowledge estimates we used to fit each GLMM could reliably predict participants' success on held-out questions, we then fit a second GLMM to the observations 
-
-
-%DIF > $$
-%DIF > \begin{aligned}
-%DIF >   \operatorname{accuracy}_{i}  &\sim \operatorname{Binomial}(n = 1, \operatorname{prob}_{\operatorname{accuracy} = \operatorname{correct}} = \widehat{P}) \\
-%DIF >     \log\left[\frac{\hat{P}}{1 - \hat{P}} \right] &=\alpha_{j[i],k[i]} \\
-%DIF > \left(
-%DIF >   \begin{array}{c}
-%DIF >     \begin{aligned}
-%DIF >       &\alpha_{j} \\
-%DIF >       &\gamma_{1j}
-%DIF >     \end{aligned}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF >   &\sim N \left(
-%DIF > \left(
-%DIF >   \begin{array}{c}
-%DIF >     \begin{aligned}
-%DIF >       &\gamma_{0}^{\alpha} + \gamma_{1k[i]}^{\alpha}(\operatorname{knowledge}) \\
-%DIF >       &\mu_{\gamma_{1j}}
-%DIF >     \end{aligned}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF > ,
-%DIF > \left(
-%DIF >   \begin{array}{cc}
-%DIF >      \sigma^2_{\alpha_{j}} & \rho_{\alpha_{j}\gamma_{1j}} \\
-%DIF >      \rho_{\gamma_{1j}\alpha_{j}} & \sigma^2_{\gamma_{1j}}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF >  \right)
-%DIF >     \text{, for participant j = 1,} \dots \text{,J} \\
-%DIF > \left(
-%DIF >   \begin{array}{c}
-%DIF >     \begin{aligned}
-%DIF >       &\alpha_{k} \\
-%DIF >       &\gamma_{1k}
-%DIF >     \end{aligned}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF >   &\sim N \left(
-%DIF > \left(
-%DIF >   \begin{array}{c}
-%DIF >     \begin{aligned}
-%DIF >       &\mu_{\alpha_{k}} \\
-%DIF >       &\mu_{\gamma_{1k}}
-%DIF >     \end{aligned}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF > ,
-%DIF > \left(
-%DIF >   \begin{array}{cc}
-%DIF >      \sigma^2_{\alpha_{k}} & \rho_{\alpha_{k}\gamma_{1k}} \\
-%DIF >      \rho_{\gamma_{1k}\alpha_{k}} & \sigma^2_{\gamma_{1k}}
-%DIF >   \end{array}
-%DIF > \right)
-%DIF >  \right)
-%DIF >     \text{, for question k = 1,} \dots \text{,K}
-%DIF > \end{aligned}
-%DIF > $$
-
 \DIFaddend \subsubsection*{Estimating the ``smoothness'' of knowledge}\label{subsec:smoothness}
 
 In the analysis reported in Figure~\ref{fig:smoothness}A, we show how

paper/changes.tex

@@ -1502,11 +1502,9 @@ \subsubsection*{Generalized linear mixed models}\label{subsec:glmm}
 Fundamental Forces}, \textit{Birth of Stars}, or general physics knowledge.
 Note that with our coding scheme, identifiers for each \texttt{question} are
 implicitly nested within levels of \texttt{lecture} and do not require explicit
-nesting in our model formula.
-
-% We then iteratively removed random effects from the maximal model until it
-% successfully converged with a full rank (i.e., non-singular) random effects
-% variance-covariance matrix.
+nesting in our model formula. We then iteratively removed random effects from 
+the maximal model until it successfully converged with a full rank (i.e., non-singular) 
+random effects variance-covariance matrix.
 
 %% JRM NOTE: do we need this next paragraph?  Commenting out for now...
 % %When inspecting the model's random effect estimates revealed multiple terms estimated at the boundary of their parameter space (i.e., variance components of 0 or correlation terms of  $\pm 1$), we found that the order in which we eliminated these terms typically did not affect which terms did and did not need to be removed in order for the model to converge to a non-degenerate solution.
@@ -1531,96 +1529,20 @@ \subsubsection*{Generalized linear mixed models}\label{subsec:glmm}
 
 To assess the predictive value of our knowledge estimates, we compared each
 GLMM's ability to discriminate between correctly and incorrectly answered
-questions to that of an analogous model that did not consider estimated
+questions to that of an analogous model that did \textit{not} consider estimated
 knowledge. Specifically, we used the same sets of observations with which we
 fit each ``full'' model to fit a second ``null'' model, with the formula:
 \[
     \mathtt{accuracy \sim (1\ \vert\ participant) + (1\ \vert\ question)}
 \]
-where the terms are as defined above.
-
-
-
-
-
-
-
-
-
-%In order to assess the predictive value of the knowledge estimates, we then fit a second set of ``null'' models to the same sets of observations used to fit our full GLMMs. These 
-%
-%
-%used the same sets of observations used to fit these GLMMs to fit a second set of ``null'' models. 
-%
-%
-%
-%Next, in order to assess the predictive value of the knowledge estimates 
-%
-%
-%
-%In order to assess the predictive value of the knowledge estimates we used to fit each GLMM, we then fit a \textit{second} model to the same data as each of the 15 
-%
-%In order to assess whether the knowledge estimates we used to fit each GLMM could reliably predict participants' success on held-out questions, we then fit a second GLMM to the observations 
-
-
-%$$
-%\begin{aligned}
-%  \operatorname{accuracy}_{i}  &\sim \operatorname{Binomial}(n = 1, \operatorname{prob}_{\operatorname{accuracy} = \operatorname{correct}} = \widehat{P}) \\
-%    \log\left[\frac{\hat{P}}{1 - \hat{P}} \right] &=\alpha_{j[i],k[i]} \\
-%\left(
-%  \begin{array}{c}
-%    \begin{aligned}
-%      &\alpha_{j} \\
-%      &\gamma_{1j}
-%    \end{aligned}
-%  \end{array}
-%\right)
-%  &\sim N \left(
-%\left(
-%  \begin{array}{c}
-%    \begin{aligned}
-%      &\gamma_{0}^{\alpha} + \gamma_{1k[i]}^{\alpha}(\operatorname{knowledge}) \\
-%      &\mu_{\gamma_{1j}}
-%    \end{aligned}
-%  \end{array}
-%\right)
-%,
-%\left(
-%  \begin{array}{cc}
-%     \sigma^2_{\alpha_{j}} & \rho_{\alpha_{j}\gamma_{1j}} \\
-%     \rho_{\gamma_{1j}\alpha_{j}} & \sigma^2_{\gamma_{1j}}
-%  \end{array}
-%\right)
-% \right)
-%    \text{, for participant j = 1,} \dots \text{,J} \\
-%\left(
-%  \begin{array}{c}
-%    \begin{aligned}
-%      &\alpha_{k} \\
-%      &\gamma_{1k}
-%    \end{aligned}
-%  \end{array}
-%\right)
-%  &\sim N \left(
-%\left(
-%  \begin{array}{c}
-%    \begin{aligned}
-%      &\mu_{\alpha_{k}} \\
-%      &\mu_{\gamma_{1k}}
-%    \end{aligned}
-%  \end{array}
-%\right)
-%,
-%\left(
-%  \begin{array}{cc}
-%     \sigma^2_{\alpha_{k}} & \rho_{\alpha_{k}\gamma_{1k}} \\
-%     \rho_{\gamma_{1k}\alpha_{k}} & \sigma^2_{\gamma_{1k}}
-%  \end{array}
-%\right)
-% \right)
-%    \text{, for question k = 1,} \dots \text{,K}
-%\end{aligned}
-%$$
+where ``\texttt{accuracy}'', ``\texttt{participant}'', and ``\texttt{question}'' are as defined above. 
+As with our full models, the null models we fit for the “All questions” version of the analysis for each quiz contained an additional term, $\mathtt{(1\ \vert\ lecture)}$, where ``\texttt{lecture}'' are as defined above. 
+We then compared each full model to its reduced (null) equivalent using a likelihood-ratio test (LRT). 
+Because the typical asymptotic $\chi^2_d$ approximation of the null distribution for the LRT statistic ($\lambda_{LR}$) is anti-conservative for models that differ in their random slope terms~\citep{GoldSimo00,ScheEtal08b,SnijBosk11}, we computed $p$-values for these tests using a parametric bootstrapping procedure~\citep{HaleHojs14}. 
+For each of 1,000 bootstraps, we used the fitted null model to simulate a sample of observations of equal size to our original sample.
+We then re-fit both the null and full models to this simulated sample and compared them via an LRT.
+This yielded a distribution of $\lambda_{LR}$ statistics we may expect to observe under our null hypothesis.
+Following~\citep{DaviHink97,NortEtal02}, we computed a corrected $p$-value for our observed $\lambda_{LR}$ as $\frac{r + 1}{n + 1}$, where $r$ is the number of simulated model comparisons that yielded a $\lambda_{LR}$ greater than or equal to our observed value and $n$ is the number of simulations we ran (1,000).
 
 \subsubsection*{Estimating the ``smoothness'' of knowledge}\label{subsec:smoothness}