Skip to content

Commit 76aa042

Browse files
MattStephansonMattStephanson
authored
[libc++] Adjust some of the [rand.dist] critical values that are too strict (#88669)
Adjust some of the [rand.dist] critical values that are too strict - Most critical values are determined empirically by running each test 51 times with a different PRNG seed and finding the smallest symmetric interval around the median that contains 90% of the sample means, variances, etc. - For the Kolmogorov-Smirnov tests, the alpha=0.1 critical value for large N is 1.224/sqrt(N). - For normally distributed variates, the sample kurtosis is distributed as Normal(0, 24/N). For N=1e5, this gives a 90% confidence interval of 0+/-0.0255. For Binomial(40, 0.25), which is approximately normal, the kurtosis is -0.0167, so the relative 90% CI is large, on the order of 0.0255/0.0167 = 153%. In most cases the distribution of the sample kurtosis isn't known analytically, but similarly large relative tolerances can be expected if the kurtosis is near zero.
1 parent caacf86 commit 76aa042

File tree

28 files changed

+192
-199
lines changed

28 files changed

+192
-199
lines changed

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.bernoulli/eval.pass.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ int main(int, char**)
6363
double x_kurtosis = (6 * sqr(d.p()) - 6 * d.p() + 1)/x_var;
6464
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
6565
assert(std::abs((var - x_var) / x_var) < 0.01);
66-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
67-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
66+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
67+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
6868
}
6969
{
7070
typedef std::bernoulli_distribution D;
@@ -99,8 +99,8 @@ int main(int, char**)
9999
double x_kurtosis = (6 * sqr(d.p()) - 6 * d.p() + 1)/x_var;
100100
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
101101
assert(std::abs((var - x_var) / x_var) < 0.01);
102-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
103-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
102+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
103+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
104104
}
105105

106106
return 0;

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.bernoulli/eval_param.pass.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ int main(int, char**)
6565
double x_kurtosis = (6 * sqr(p.p()) - 6 * p.p() + 1)/x_var;
6666
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
6767
assert(std::abs((var - x_var) / x_var) < 0.01);
68-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
69-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
68+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
69+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
7070
}
7171
{
7272
typedef std::bernoulli_distribution D;
@@ -103,8 +103,8 @@ int main(int, char**)
103103
double x_kurtosis = (6 * sqr(p.p()) - 6 * p.p() + 1)/x_var;
104104
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
105105
assert(std::abs((var - x_var) / x_var) < 0.01);
106-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
107-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
106+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
107+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
108108
}
109109

110110
return 0;

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.bin/eval.PR44847.pass.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ int main(int, char**) {
163163
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
164164
assert(std::abs((var - x_var) / x_var) < 0.01);
165165
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
166-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
166+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.06);
167167

168168
return 0;
169169
}

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.bin/eval.pass.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ void test1() {
6868
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
6969
assert(std::abs((var - x_var) / x_var) < 0.01);
7070
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
71-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
71+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
7272
}
7373

7474
template <class T>
@@ -109,8 +109,8 @@ void test2() {
109109
double x_kurtosis = (1-6*d.p()*(1-d.p())) / x_var;
110110
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
111111
assert(std::abs((var - x_var) / x_var) < 0.01);
112-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
113-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
112+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
113+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
114114
}
115115

116116
template <class T>
@@ -151,8 +151,8 @@ void test3() {
151151
double x_kurtosis = (1-6*d.p()*(1-d.p())) / x_var;
152152
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
153153
assert(std::abs((var - x_var) / x_var) < 0.01);
154-
assert(std::abs((skew - x_skew) / x_skew) < 0.03);
155-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
154+
assert(std::abs((skew - x_skew) / x_skew) < 0.07);
155+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 2.0);
156156
}
157157

158158
template <class T>
@@ -292,7 +292,7 @@ void test6() {
292292
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
293293
assert(std::abs((var - x_var) / x_var) < 0.01);
294294
assert(std::abs(skew - x_skew) < 0.02);
295-
assert(std::abs(kurtosis - x_kurtosis) < 0.01);
295+
assert(std::abs(kurtosis - x_kurtosis) < 0.03);
296296
}
297297

298298
template <class T>

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.bin/eval_param.pass.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ int main(int, char**)
7272
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
7373
assert(std::abs((var - x_var) / x_var) < 0.01);
7474
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
75-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.04);
75+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
7676
}
7777
{
7878
typedef std::binomial_distribution<> D;
@@ -113,8 +113,8 @@ int main(int, char**)
113113
double x_kurtosis = (1-6*p.p()*(1-p.p())) / x_var;
114114
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
115115
assert(std::abs((var - x_var) / x_var) < 0.01);
116-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
117-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
116+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
117+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
118118
}
119119
{
120120
typedef std::binomial_distribution<> D;
@@ -155,8 +155,8 @@ int main(int, char**)
155155
double x_kurtosis = (1-6*p.p()*(1-p.p())) / x_var;
156156
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
157157
assert(std::abs((var - x_var) / x_var) < 0.01);
158-
assert(std::abs((skew - x_skew) / x_skew) < 0.04);
159-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
158+
assert(std::abs((skew - x_skew) / x_skew) < 0.07);
159+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 2.0);
160160
}
161161

162162
return 0;

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.geo/eval.pass.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ void test1() {
7777
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
7878
assert(std::abs((var - x_var) / x_var) < 0.01);
7979
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
80-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
80+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
8181
}
8282

8383
template <class T>
@@ -161,7 +161,7 @@ void test3() {
161161
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
162162
assert(std::abs((var - x_var) / x_var) < 0.01);
163163
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
164-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
164+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
165165
}
166166

167167
template <class T>
@@ -203,7 +203,7 @@ void test4() {
203203
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
204204
assert(std::abs((var - x_var) / x_var) < 0.01);
205205
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
206-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
206+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
207207
}
208208

209209
template <class T>
@@ -245,7 +245,7 @@ void test5() {
245245
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
246246
assert(std::abs((var - x_var) / x_var) < 0.01);
247247
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
248-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
248+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
249249
}
250250

251251
template <class T>
@@ -284,8 +284,8 @@ void test6() {
284284
double x_var = x_mean / d.p();
285285
double x_skew = (2 - d.p()) / std::sqrt((1 - d.p()));
286286
double x_kurtosis = 6 + sqr(d.p()) / (1 - d.p());
287-
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
288-
assert(std::abs((var - x_var) / x_var) < 0.01);
287+
assert(std::abs((mean - x_mean) / x_mean) < 0.02);
288+
assert(std::abs((var - x_var) / x_var) < 0.02);
289289
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
290290
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
291291
}

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.geo/eval_param.pass.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ int main(int, char**)
7272
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
7373
assert(std::abs((var - x_var) / x_var) < 0.01);
7474
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
75-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
75+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
7676
}
7777
{
7878
typedef std::geometric_distribution<> D;
@@ -156,7 +156,7 @@ int main(int, char**)
156156
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
157157
assert(std::abs((var - x_var) / x_var) < 0.01);
158158
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
159-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
159+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
160160
}
161161

162162
return 0;

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.negbin/eval.pass.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ void test1() {
7474
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
7575
assert(std::abs((var - x_var) / x_var) < 0.01);
7676
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
77-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.02);
77+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
7878
}
7979

8080
template <class T>
@@ -115,8 +115,8 @@ void test2() {
115115
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
116116
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
117117
assert(std::abs((var - x_var) / x_var) < 0.01);
118-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
119-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
118+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
119+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.1);
120120
}
121121

122122
template <class T>
@@ -157,8 +157,8 @@ void test3() {
157157
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
158158
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
159159
assert(std::abs((var - x_var) / x_var) < 0.01);
160-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
161-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
160+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
161+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
162162
}
163163

164164
template <class T>
@@ -243,8 +243,8 @@ void test5() {
243243
double x_kurtosis = 6. / d.k() + sqr(d.p()) / (d.k() * (1 - d.p()));
244244
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
245245
assert(std::abs((var - x_var) / x_var) < 0.01);
246-
assert(std::abs((skew - x_skew) / x_skew) < 0.04);
247-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.05);
246+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
247+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.3);
248248
}
249249

250250
template <class T>

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.bern/rand.dist.bern.negbin/eval_param.pass.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ int main(int, char**)
7272
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
7373
assert(std::abs((var - x_var) / x_var) < 0.01);
7474
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
75-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
75+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
7676
}
7777
{
7878
typedef std::negative_binomial_distribution<> D;
@@ -113,8 +113,8 @@ int main(int, char**)
113113
double x_kurtosis = 6. / p.k() + sqr(p.p()) / (p.k() * (1 - p.p()));
114114
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
115115
assert(std::abs((var - x_var) / x_var) < 0.01);
116-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
117-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.01);
116+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
117+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.1);
118118
}
119119
{
120120
typedef std::negative_binomial_distribution<> D;
@@ -155,8 +155,8 @@ int main(int, char**)
155155
double x_kurtosis = 6. / p.k() + sqr(p.p()) / (p.k() * (1 - p.p()));
156156
assert(std::abs((mean - x_mean) / x_mean) < 0.01);
157157
assert(std::abs((var - x_var) / x_var) < 0.01);
158-
assert(std::abs((skew - x_skew) / x_skew) < 0.01);
159-
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.03);
158+
assert(std::abs((skew - x_skew) / x_skew) < 0.02);
159+
assert(std::abs((kurtosis - x_kurtosis) / x_kurtosis) < 0.08);
160160
}
161161

162162
return 0;

‎libcxx/test/std/numerics/rand/rand.dist/rand.dist.norm/rand.dist.norm.cauchy/eval.pass.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ int main(int, char**)
4545
u.push_back(d(g));
4646
std::sort(u.begin(), u.end());
4747
for (int i = 0; i < N; ++i)
48-
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
48+
assert(std::abs(f(u[i], a, b) - double(i) / N) < .0013);
4949
}
5050
{
5151
typedef std::cauchy_distribution<> D;
@@ -60,7 +60,7 @@ int main(int, char**)
6060
u.push_back(d(g));
6161
std::sort(u.begin(), u.end());
6262
for (int i = 0; i < N; ++i)
63-
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
63+
assert(std::abs(f(u[i], a, b) - double(i) / N) < .0013);
6464
}
6565
{
6666
typedef std::cauchy_distribution<> D;
@@ -75,7 +75,7 @@ int main(int, char**)
7575
u.push_back(d(g));
7676
std::sort(u.begin(), u.end());
7777
for (int i = 0; i < N; ++i)
78-
assert(std::abs(f(u[i], a, b) - double(i)/N) < .001);
78+
assert(std::abs(f(u[i], a, b) - double(i) / N) < .0013);
7979
}
8080

8181
return 0;

0 commit comments

Comments
 (0)