-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathWeek3.html
More file actions
1691 lines (1549 loc) · 103 KB
/
Week3.html
File metadata and controls
1691 lines (1549 loc) · 103 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
<meta charset="utf-8">
<meta name="generator" content="quarto-1.7.32">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<title>3 Integrity: Challenging questionable research practices – Open Research</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
width: 0.8em;
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align: middle;
}
</style>
<script src="site_libs/quarto-nav/quarto-nav.js"></script>
<script src="site_libs/quarto-nav/headroom.min.js"></script>
<script src="site_libs/clipboard/clipboard.min.js"></script>
<script src="site_libs/quarto-search/autocomplete.umd.js"></script>
<script src="site_libs/quarto-search/fuse.min.js"></script>
<script src="site_libs/quarto-search/quarto-search.js"></script>
<meta name="quarto:offset" content="./">
<link href="./Week4.html" rel="next">
<link href="./Week2.html" rel="prev">
<script src="site_libs/quarto-html/quarto.js" type="module"></script>
<script src="site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="site_libs/quarto-html/popper.min.js"></script>
<script src="site_libs/quarto-html/tippy.umd.min.js"></script>
<script src="site_libs/quarto-html/anchor.min.js"></script>
<link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
<link href="site_libs/quarto-html/quarto-syntax-highlighting-37eea08aefeeee20ff55810ff984fec1.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="site_libs/bootstrap/bootstrap.min.js"></script>
<link href="site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="site_libs/bootstrap/bootstrap-7e4aeafd2f2715da386b171fb548fac0.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
<link href="site_libs/quarto-contrib/popover-glossary/popover-glossary.css" rel="stylesheet">
<script src="site_libs/quarto-contrib/videojs/video.min.js"></script>
<link href="site_libs/quarto-contrib/videojs/video-js.css" rel="stylesheet">
<script id="quarto-search-options" type="application/json">{
"location": "sidebar",
"copy-button": false,
"collapse-after": 3,
"panel-placement": "start",
"type": "textbox",
"limit": 50,
"keyboard-shortcut": [
"f",
"/",
"s"
],
"show-item-context": false,
"language": {
"search-no-results-text": "No results",
"search-matching-documents-text": "matching documents",
"search-copy-link-title": "Copy link to search",
"search-hide-matches-text": "Hide additional matches",
"search-more-match-text": "more match in this document",
"search-more-matches-text": "more matches in this document",
"search-clear-button-title": "Clear",
"search-text-placeholder": "",
"search-detached-cancel-button-title": "Cancel",
"search-submit-button-title": "Submit",
"search-label": "Search"
}
}</script>
<link rel="shortcut icon" href="assets/forrt_icon.png">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css">
<style>
div.callout-caution.callout {
border-left-color: #ff9a00;
}
div.callout-caution.callout-style-default > .callout-header {
background-color: rgba(255, 154, 0, 0.13);
}
div.callout-caution .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-caution.callout-style-default .callout-icon::before, div.callout-caution.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f071' !important;
background-image: none;
}
div.callout-red.callout {
border-left-color: #8e0000;
}
div.callout-red.callout-style-default > .callout-header {
background-color: rgba(142, 0, 0, 0.13);
}
div.callout-red .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-red.callout-style-default .callout-icon::before, div.callout-red.callout-titled .callout-icon::before {
background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="%238e0000" class="bi bi-exclamation-triangle" viewBox="0 0 16 16"><path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/></svg>');
}
div.callout-yellow.callout {
border-left-color: #86754a;
}
div.callout-yellow.callout-style-default > .callout-header {
background-color: rgba(134, 117, 74, 0.13);
}
div.callout-yellow .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-yellow.callout-style-default .callout-icon::before, div.callout-yellow.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f05a' !important;
background-image: none;
}
div.callout-green.callout {
border-left-color: #bad6c2;
}
div.callout-green.callout-style-default > .callout-header {
background-color: rgba(186, 214, 194, 0.13);
}
div.callout-green .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-green.callout-style-default .callout-icon::before, div.callout-green.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f0eb' !important;
background-image: none;
}
</style>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="include/webex.css">
</head>
<body class="nav-sidebar floating quarto-light">
<div id="quarto-search-results"></div>
<header id="quarto-header" class="headroom fixed-top">
<nav class="quarto-secondary-nav">
<div class="container-fluid d-flex">
<button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
<i class="bi bi-layout-text-sidebar-reverse"></i>
</button>
<nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./Week1.html">Chapters</a></li><li class="breadcrumb-item"><a href="./Week3.html"><span class="chapter-number">3</span> <span class="chapter-title">Integrity: Challenging questionable research practices</span></a></li></ol></nav>
<a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
</a>
<button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();">
<i class="bi bi-search"></i>
</button>
</div>
</nav>
</header>
<!-- content -->
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-full">
<!-- sidebar -->
<nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation floating overflow-auto">
<div class="pt-lg-2 mt-2 text-left sidebar-header sidebar-header-stacked">
<a href="./index.html" class="sidebar-logo-link">
<img src="https://forrt.org/img/FORRT.svg" alt="" class="sidebar-logo py-0 d-lg-inline d-none">
</a>
<div class="sidebar-title mb-0 py-0">
<a href="./">Open Research</a>
<div class="sidebar-tools-main">
<a href="https://github.com/forrtproject/Open-Research-course" title="Source Code" class="quarto-navigation-tool px-1" aria-label="Source Code"><i class="bi bi-github"></i></a>
</div>
</div>
</div>
<div class="mt-2 flex-shrink-0 align-items-center">
<div class="sidebar-search">
<div id="quarto-search" class="" title="Search"></div>
</div>
</div>
<div class="sidebar-menu-container">
<ul class="list-unstyled mt-1">
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
<span class="menu-text">Overview</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./index.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Overview</span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
<span class="menu-text">Chapters</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week1.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">1</span> <span class="chapter-title">What is Open Research</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week2.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">2</span> <span class="chapter-title">Transparency: As open as possible</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week3.html" class="sidebar-item-text sidebar-link active">
<span class="menu-text"><span class="chapter-number">3</span> <span class="chapter-title">Integrity: Challenging questionable research practices</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week4.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">4</span> <span class="chapter-title">Documenting Decisions Transparently</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week5.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">5</span> <span class="chapter-title">Integrity: Supporting robust interpretations</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week6.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">6</span> <span class="chapter-title">Accessibility: Making your research accessible online</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week7.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">7</span> <span class="chapter-title">Accessibility: Academic privilege and diversity</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week8.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">8</span> <span class="chapter-title">Committing to open research</span></span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true">
<span class="menu-text">Appendices</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./glossary.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Glossary</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./reference.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Reference</span></a>
</div>
</li>
</ul>
</li>
</ul>
</div>
</nav>
<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
<!-- margin-sidebar -->
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
<nav id="TOC" role="doc-toc" class="toc-active">
<h2 id="toc-title">Table of contents</h2>
<ul>
<li><a href="#replicability" id="toc-replicability" class="nav-link active" data-scroll-target="#replicability"><span class="header-section-number">3.1</span> Replicability</a></li>
<li><a href="#replication-studies" id="toc-replication-studies" class="nav-link" data-scroll-target="#replication-studies"><span class="header-section-number">3.2</span> Replication studies</a>
<ul>
<li><a href="#limits-to-replication" id="toc-limits-to-replication" class="nav-link" data-scroll-target="#limits-to-replication"><span class="header-section-number">3.2.1</span> Limits to replication</a>
<ul class="collapse">
<li><a href="#activity-1" id="toc-activity-1" class="nav-link" data-scroll-target="#activity-1">Activity 1</a></li>
</ul></li>
</ul></li>
<li><a href="#the-replication-crisis" id="toc-the-replication-crisis" class="nav-link" data-scroll-target="#the-replication-crisis"><span class="header-section-number">3.3</span> The replication crisis</a></li>
<li><a href="#questionable-research-practices" id="toc-questionable-research-practices" class="nav-link" data-scroll-target="#questionable-research-practices"><span class="header-section-number">3.4</span> Questionable research practices</a></li>
<li><a href="#writing-transparently" id="toc-writing-transparently" class="nav-link" data-scroll-target="#writing-transparently"><span class="header-section-number">3.5</span> Writing transparently</a>
<ul>
<li><a href="#activity-2-what-not-to-do" id="toc-activity-2-what-not-to-do" class="nav-link" data-scroll-target="#activity-2-what-not-to-do">Activity 2: What not to do!</a></li>
</ul></li>
<li><a href="#generalisability" id="toc-generalisability" class="nav-link" data-scroll-target="#generalisability"><span class="header-section-number">3.6</span> Generalisability</a></li>
<li><a href="#studying-generalisability" id="toc-studying-generalisability" class="nav-link" data-scroll-target="#studying-generalisability"><span class="header-section-number">3.7</span> Studying generalisability</a>
<ul>
<li><a href="#activity-3" id="toc-activity-3" class="nav-link" data-scroll-target="#activity-3">Activity 3</a></li>
</ul></li>
<li><a href="#quiz" id="toc-quiz" class="nav-link" data-scroll-target="#quiz"><span class="header-section-number">3.8</span> Quiz</a></li>
<li><a href="#summary" id="toc-summary" class="nav-link" data-scroll-target="#summary"><span class="header-section-number">3.9</span> Summary</a></li>
</ul>
<div class="toc-actions"><ul><li><a href="https://github.com/forrtproject/Open-Research-course/issues" class="toc-action"><i class="bi bi-github"></i>Report an issue</a></li><li><a href="https://github.com/forrtproject/Open-Research-course/edit/main/Week3.qmd" class="toc-action"><i class="bi empty"></i>Edit this page</a></li></ul></div></nav>
</div>
<!-- main -->
<main class="content column-body" id="quarto-document-content">
<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./Week1.html">Chapters</a></li><li class="breadcrumb-item"><a href="./Week3.html"><span class="chapter-number">3</span> <span class="chapter-title">Integrity: Challenging questionable research practices</span></a></li></ol></nav>
<div class="quarto-title">
<h1 class="title"><span class="chapter-number">3</span> <span class="chapter-title">Integrity: Challenging questionable research practices</span></h1>
</div>
<div class="quarto-title-meta column-body">
</div>
</header>
<p>In Week 1, we broke down the idea of open research into three key facets that make research open: transparency, integrity and accessibility. We’re now going to take a deeper dive into integrity: how trustworthy a study is.</p>
<p>In this week, you will learn how to recognise and avoid questionable research practices. You will discover why it is often important to be able to replicate other researchers’ findings, and how to go about doing this. You will experience an important test for the integrity of a piece of research: its <a class="glossary-ref" data-glossary-term="robustness" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3JvYnVzdG5lc3MnIHJvbGU9J3Rvb2x0aXAnPjxwPlJlZmVycyB0byB0aGUgc3RyZW5ndGggYW5kIHJlbGlhYmlsaXR5IG9mIHJlc3VsdHMuPC9wPjwvZGl2Pg==" href="javascript:void(0);">robustness</a>.</p>
<section id="replicability" class="level2" data-number="3.1">
<h2 data-number="3.1" class="anchored" data-anchor-id="replicability"><span class="header-section-number">3.1</span> Replicability</h2>
<p>There may be cases where you don’t expect to get the same results if you conduct the same study again. For example, if a study is based around a specific political event, it may be difficult or even impossible to replicate. But in many other types of investigation, we would expect to be able to get the same results when we run the same study again.</p>
<p>In Week 2, we talked about <a class="glossary-ref" data-glossary-term="reproducibility" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3JlcHJvZHVjaWJpbGl0eScgcm9sZT0ndG9vbHRpcCc+QQoKc3R1ZHkKCmlzCgpyZXByb2R1Y2libGUKCmlmCgp5b3UKCmFyZQoKYWJsZQoKdG8KCmdldAoKdGhlCgpzYW1lCgpyZXN1bHRzCgp3aGVuCgpjb25kdWN0aW5nCgp0aGUKCnNhbWUKCmFuYWx5c2VzCgpvbgoKdGhlCgpzYW1lCgpkYXRhCgphcwoKdGhlCgpvcmlnaW5hbAoKc3R1ZHkuPC9kaXY+" href="javascript:void(0);">reproducibility</a> – being able to get the same results when conducting the same analyses on the same data as the original study. <a class="glossary-ref" data-glossary-term="replicability" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3JlcGxpY2FiaWxpdHknIHJvbGU9J3Rvb2x0aXAnPjxwPkEgc3R1ZHkgaXMgcmVwbGljYWJsZSBpZiB5b3UgYXJlIGFibGUgdG8gY29uZHVjdCB0aGUgc2FtZSBzdHVkeQphZ2FpbiwgZ2VuZXJhdGUgbmV3IGRhdGEsIGFuZCBzdGlsbCBnZXQgdGhlIHNhbWUgcmVzdWx0cyBhcyB0aGUgb3JpZ2luYWwKc3R1ZHkuPC9wPjwvZGl2Pg==" href="javascript:void(0);">Replicability</a> is similar, in that it’s about getting the same results as the original study when running the same analyses, however, the difference is that now these analyses are run on new data. So, replication means conducting the same study again, and seeing if you get the same results.</p>
<p>Replication studies are deliberate attempts to do this. But what does the ‘same study’ mean? There are always going to be differences between the original study and the replication study. Replication studies vary on a spectrum from ‘direct’ to ‘conceptual’. <a class="glossary-ref" data-glossary-term="directreplications" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2RpcmVjdHJlcGxpY2F0aW9ucycgcm9sZT0ndG9vbHRpcCc+PHA+QSB0eXBlIG9mIHJlcGxpY2F0aW9uIHN0dWR5IHdoaWNoIGFpbXMgdG8gc3RheSBhcyBjbG9zZSB0byB0aGUKb3JpZ2luYWwgc3R1ZHkgYXMgcG9zc2libGUuPC9wPjwvZGl2Pg==" href="javascript:void(0);">Direct replications</a> try to stay as close to the original study as possible, whereas <a class="glossary-ref" data-glossary-term="conceptualreplications" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2NvbmNlcHR1YWxyZXBsaWNhdGlvbnMnIHJvbGU9J3Rvb2x0aXAnPjxwPkEgdHlwZSBvZiByZXBsaWNhdGlvbiBzdHVkeSB3aGljaCBhaW1zIHRvIHZhcnkgc29tZSBhc3BlY3Qgb2YgdGhlCm9yaWdpbmFsIHN0dWR5LCBpbiBvcmRlciB0byBiZXR0ZXIgdW5kZXJzdGFuZCB0aGUgdW5kZXJseWluZwpwaGVub21lbm9uLjwvcD48L2Rpdj4=" href="javascript:void(0);">conceptual replications</a> purposefully vary some aspects to better understand the underlying phenomenon. Here are some examples, from most direct (the first) to most conceptual (the last):</p>
<ul>
<li>A researcher makes a surprising finding in their research. To test whether they should rely on this result, they conduct a replication immediately after, using all the same materials and the same participant pool.</li>
<li>A researcher wants to replicate a study they’ve read about. The study is much older (from the 1990s), when open materials were not common. They only have the methods described in the original short paper to refer to, so they interpret these as best they can.</li>
<li>A researcher wants to replicate a study they’ve read about. They don’t think the original study was well-designed, but they think the hypothesis is interesting so they design a new study testing the same hypothesis but in a different way.</li>
</ul>
<p>Now let’s dig deeper into the process of designing a replication study.</p>
</section>
<section id="replication-studies" class="level2" data-number="3.2">
<h2 data-number="3.2" class="anchored" data-anchor-id="replication-studies"><span class="header-section-number">3.2</span> Replication studies</h2>
<p>In the next video, psychologist Priya Silverstein talks about their first forays into conducting a replication study, and lessons learned. As you watch the video, think about what Priya’s results tell us about the process of running a good replication study.</p>
<p></p><div class="quarto-video"><video id="video_shortcode_videojs_video1" class="video-js vjs-default-skin vjs-fluid" controls="" preload="auto" data-setup="{}" title=""><source src="https://www.open.edu/openlearncreate/pluginfile.php/828891/mod_oucontent/oucontent/86995/9d2c3b0a/8b3f7881/vid_3_1.mp4"></video></div> <a href="https://www.open.edu/openlearncreate/pluginfile.php/828891/mod_oucontent/oucontent/86995/9d2c3b0a/8b3f7881/vid_3_1.mp4?forcedownload=1" target="_blank">Download the video</a><p></p>
<div class="callout callout-style-default callout-green callout-titled">
<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-1-contents" aria-controls="callout-1" aria-expanded="false" aria-label="Toggle callout">
<div class="callout-icon-container">
<i class="callout-icon"></i>
</div>
<div class="callout-title-container flex-fill">
Video transcript
</div>
<div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
</div>
<div id="callout-1" class="callout-1-contents callout-collapse collapse">
<div class="callout-body-container callout-body">
<p>Hi everyone, my name’s Priya Silverstein and I’m a post-doctoral researcher for the Psychological Science Accelerator, and I’m also the author for this course. My pronouns are ‘they, them’.</p>
<p>As part of my PhD, I ran my first replication study. It wasn’t meant to be a big part of my PhD, but it ended up being one of the biggest parts!</p>
<p>I thought that before starting any of my own original research, it would make more sense to start with a replication study. However it wasn’t that simple, so when I ran the replication study, surprisingly, we got a null result, and I was a bit confused about why this might be. So the first thing that I did was I contacted the original researchers to ask them what they thought might be the problem.</p>
<p>They got back to me and they said that they thought it was because of some differences between the stimuli, so the things that I’d shown in my study versus the things that they showed in the original. And some of these differences were things I couldn’t have known, because they didn’t outline the specifics of that in their original paper.</p>
<p>I made some edits to the protocol to the way that I was going to run the study.</p>
<p>And then I thought, okay, now that I’ve had approval from the original authors this new version should be able to replicate the original study. So I ran it again and surprisingly I still wasn’t able to replicate the result.</p>
<p>Erm and so this was quite disappointing, both for me and the original authors, because it meant that I wasn’t able to find the same thing that they did.</p>
<p>So… This was my first experience of replications. And you might think that that was enough to put me off doing any more, but instead, quite the opposite. I ended up realising how important replications are.</p>
<p>So yeah, ever since starting with that first replication study as part of my PhD, I’ve now kind of made that my specialty.</p>
<p>My advice for anyone who would be conducting their own replication study comes from some of the mistakes that I made as part of that first replication study that I did.</p>
<p>So my first piece of advice would be to always contact the authors before you begin your replication study.</p>
<p>I think I was a bit naïve, and thought if I just follow what’s written in the paper then how can I go wrong? But papers don’t have enough space to include everything about a study that you would need to know in order to conduct a good replication.</p>
<p>So I’d recommend talking to the original authors, coming to an agreement with them, making sure that they agree that the protocol that you’ve proposed, they would agree that’s a good faith replication attempt of their study.</p>
<p>Another thing that I did wrong is that I only collected the same number of participants as in the original study, for my replication, because I thought that was more ‘replication-y’, because it was the same amount of participants as the original. But now, after learning more about both replication studies, but also sample size more generally, I would really recommend to go with a much larger sample than the original study that you’re replicating.</p>
<p>And this is just so that you can be a little bit more sure about what your findings mean. So, in my study, I wasn’t able to replicate the same result as the original authors, but this could just be because the true effect size that’s in the world for that effect that I was looking at might be smaller than what they kind of measured in the original study.</p>
<p>If I had used a much larger number of participants, if I still wasn’t able to replicate the study, we could be a bit more sure that it wasn’t just because of low sample size.</p>
<p>I ran my study just as a normal study where we finished the entire study and then submitted it to a journal for publication. And I was lucky that it was successful in getting published.</p>
<p>But it could have been a lot harder for me to publish, which would have been a bit disappointing and taken a lot of time. So what I would recommend instead is submitting any replication study as a registered report.</p>
<p>A registered report is essentially where your paper gets peer-reviewed before you have collected data. So the peer reviewers say whether your protocol makes sense, recommend any suggested changes, and then once they’ve accepted it, the journal agrees to accept your study, regardless of what the outcome is. So that would be my third piece of advice.</p>
</div>
</div>
</div>
<p><strong>Video 2: Conducting a replication study</strong></p>
<p>Write your comments on what Priya advises. <em>Allow about 10 minutes.</em> When you are ready, see our comments.</p>
<details>
<summary>Show / Hide Discussion </summary>
<p>
<br>
<strong>Advice from Priya</strong>
<br>
Priya suggests getting in touch with the authors of the original study and asking for more detail than a published paper provides. Using a larger sample size increases confidence that your findings do (or don’t) support those of the previous study. Priya also recommends submitting a registered report, to increase your chances of getting published.
</p>
</details>
<section id="limits-to-replication" class="level3" data-number="3.2.1">
<h3 data-number="3.2.1" class="anchored" data-anchor-id="limits-to-replication"><span class="header-section-number">3.2.1</span> Limits to replication</h3>
<p>There are fields and methodologies where the value of replication is hotly debated. For instance:</p>
<ul>
<li><p>Some argue that replication should be encouraged in qualitative research, whereas others argue that there are still open questions about whether replication is possible, desirable, or even aligned with the fundamental principles of qualitative research.</p></li>
<li><p>Economics has had a long history with replication studies, but not under this name. In economics, replication often takes place as ‘robustness checks’, where researchers test if their results hold when they use different datasets.</p></li>
<li><p>Research in the humanities is primarily interpretive and context-specific, focusing on understanding human experiences, cultures, texts, and historical events. This interpretive nature makes exact replication more challenging.</p></li>
</ul>
<p>It is important to think carefully about whether replication makes sense for your field and methodology.</p>
<p>If you are working in a field where replication is important, and if your study replicates the one you are trying to replicate, you can be pretty confident about the result.</p>
<p>But what does it mean if, like Priya’s first attempts, your study does not replicate? One explanation could be that the original result was a <a class="glossary-ref" data-glossary-term="falsepositive" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2ZhbHNlcG9zaXRpdmUnIHJvbGU9J3Rvb2x0aXAnPjxwPkFuIGVycm9yIHRoYXQgb2NjdXJzIHdoZW4gYSByZXNlYXJjaGVyIGJlbGlldmVzIHRoYXQgdGhlcmUgaXMgYQpnZW51aW5lIGVmZmVjdCBvciBkaWZmZXJlbmNlIHdoZW4gdGhlcmUgaXMgbm90IChlLmcuwqBhIHBlcnNvbiBoYXMgYQpwb3NpdGl2ZSBDb3ZpZCB0ZXN0IGFsdGhvdWdoIHRoZXkgZG8gbm90IGhhdmUgQ292aWQpLjwvcD48L2Rpdj4=" href="javascript:void(0);">false positive</a> , and so the failed replication is a ‘true negative’. Another explanation is that the replication result was a <a class="glossary-ref" data-glossary-term="falsenegative" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2ZhbHNlbmVnYXRpdmUnIHJvbGU9J3Rvb2x0aXAnPjxwPkFuIGVycm9yIHRoYXQgb2NjdXJzIHdoZW4gYSByZXNlYXJjaGVyIGJlbGlldmVzIHRoYXQgdGhlcmUgaXMgbm8KZWZmZWN0IG9yIGRpZmZlcmVuY2UsIHdoZW4gYWN0dWFsbHkgdGhlcmUgaXMgKGUuZy7CoGEgcGVyc29uIGhhcyBhCm5lZ2F0aXZlIENvdmlkIHRlc3QgYWx0aG91Z2ggdGhleSBkbyBoYXZlIENvdmlkKS48L3A+PC9kaXY+" href="javascript:void(0);">false negative</a>, and that the original study was a ‘true positive’. It’s also possible that differences between the two studies are responsible for the different results.</p>
<section id="activity-1" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-1">Activity 1</h4>
<p><em>Allow about 10 minutes.</em></p>
<p>This activity relates to our examples of typical direct and conceptual replication studies. By way of reminder:</p>
<p><strong>Researcher A</strong> finds a surprising finding in their research. To test whether they should rely on this result, they conduct a replication immediately after, using all the same materials and the same participant pool. This is a direct replication.</p>
<p><strong>Researcher B</strong> wants to replicate a study they’ve read about. They don’t think the original study was well-designed, but they think the hypothesis is interesting, so they design a new study testing the same hypothesis but in a different way. This is a conceptual replication.</p>
<p>Now imagine these two researchers both carry out their studies. List the reasons why each of these two researchers may not replicate the original result.</p>
<details>
<summary>Show / Hide Discussion </summary>
<p>
<br>
<strong>Why were the two researchers unable to replicate the original results?</strong><br>
You might have listed:
</p><ul>
<li> The original result was a false positive</li>
<li> The replication result is a false negative</li>
<li> There are important differences between the original study and the replication study:</li>
<ul>
<li style="list-style-type: none;">a. These could be small changes that researchers didn’t think should be important but that turned out to be (e.g.: which brand of a specific chemical was used).</li>
<li style="list-style-type: none;">b. It could be that the replication researcher didn’t realise these were differences because there wasn’t enough detail in the original paper to be able to work out how everything had been done.</li>
<li style="list-style-type: none;">c. The replication researcher might know they’re making a change from the original protocol, but approve this change because theoretically it shouldn’t make a difference to the result.</li>
</ul>
</ul>
<p></p>
</details>
</section>
</section>
</section>
<section id="the-replication-crisis" class="level2" data-number="3.3">
<h2 data-number="3.3" class="anchored" data-anchor-id="the-replication-crisis"><span class="header-section-number">3.3</span> The replication crisis</h2>
<p>This section will highlight some of the issues around replication in quantitative research. Replication is possible in qualitative research, and many qualitative researchers see the value of replication. So if you are a qualitative researcher, this section is still relevant to you. It will allow you to explore key issues faced by quantitative colleagues, learn how to read quantitative research papers more critically, and think about whether these issues could also apply to qualitative research, albeit manifested differently.</p>
<p>If we consider relatively direct replications, using the same materials as the original authors but conducted by different researchers, what percentage of published results do you imagine would replicate? It would be tempting to think that most published research findings are true, and therefore that a replication of a published research finding would be pretty likely to find the same result. However, researchers have found these percentages of findings could not be replicated:</p>
<ul>
<li>Psychology: up to 60%</li>
<li>Cancer biology: up to 55%</li>
<li>Economics: up to 40%</li>
<li>Philosophy: up to 30%</li>
</ul>
<p>The number of studies that could not be replicated was much higher than expected in certain fields, which has led some to refer to this as a ‘replication crisis’.</p>
<p>Why is it that so many quantitative studies cannot be replicated? It’s complicated!</p>
<p>Previously, you learned the three classifications of failed replications: the original result was a false positive, the replication result was a false negative, or differences between the two studies could have been responsible for the different results. However, these three interpretations are not all as likely as each other. There are ways to try and work out which of these are most likely.</p>
<p>The original result being a false positive is more likely than you would think. Researchers often do not publish all the research that they do. As a researcher, there is an incentive to publish papers in ‘high impact’ journals (journals that are regarded highly in the researcher’s discipline, and that publish papers that receive a high number of citations). Historically, it has been harder to publish negative (null) results than positive (statistically significant) results, as journals have prioritised headline-grabbing results that confirm popular or contemporary positions. This has been the case for all journals, but especially high-impact ones.</p>
<p>This means that researchers have an incentive to get positive results in their research and can feel disappointed, stressed, and even ashamed if they don’t get a significant result. This can entice them to turn to questionable research practices, to increase the likelihood of a false positive result.</p>
</section>
<section id="questionable-research-practices" class="level2" data-number="3.4">
<h2 data-number="3.4" class="anchored" data-anchor-id="questionable-research-practices"><span class="header-section-number">3.4</span> Questionable research practices</h2>
<p>Yes, you read the end of the previous section correctly. There are questionable research practices that researchers may feel pressurised to use. Here are some examples:</p>
<ul>
<li><a class="glossary-ref" data-glossary-term="phack1" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3BoYWNrMScgcm9sZT0ndG9vbHRpcCc+PHA+KEZyb20gRk9SUlQgR2xvc3NhcnkpIEV4cGxvaXRpbmcgdGVjaG5pcXVlcyB0aGF0IG1heSBhcnRpZmljaWFsbHkKaW5jcmVhc2UgdGhlIGxpa2VsaWhvb2Qgb2Ygb2J0YWluaW5nIGEgc3RhdGlzdGljYWxseSBzaWduaWZpY2FudCByZXN1bHQKYnkgbWVldGluZyB0aGUgc3RhbmRhcmQgc3RhdGlzdGljYWwgc2lnbmlmaWNhbmNlIGNyaXRlcmlvbiAodHlwaWNhbGx5IM6xCj0gLjA1KS48L3A+PC9kaXY+" href="javascript:void(0);">P-hacking</a>: in quantitative research, p-hacking means exploiting techniques that increase the likelihood of obtaining a statistically significant result, for example by performing multiple analyses, or stopping data collection once a significant p-value is reached.</li>
<li><a class="glossary-ref" data-glossary-term="selectivereporting" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3NlbGVjdGl2ZXJlcG9ydGluZycgcm9sZT0ndG9vbHRpcCc+PHA+UmVzZWFyY2hlcnMgYXJlIHNlbGVjdGl2ZSByZXBvcnRpbmcgaWYgdGhlaXIgcmVzdWx0cyBhcmUgZGVsaWJlcmF0ZWx5Cm5vdCBmdWxseSBvciBhY2N1cmF0ZWx5IHJlcG9ydGVkLCBpbiBvcmRlciB0byBzdXBwcmVzcyBuZWdhdGl2ZSBvcgp1bmRlc2lyYWJsZSBmaW5kaW5ncy48L3A+PC9kaXY+" href="javascript:void(0);">Selective reporting</a>: when results from research are deliberately not fully or accurately reported, in order to suppress negative or undesirable findings. For example, researchers might run two analyses but only report the one with significant findings, or be selective in what results are included in a report aimed at particular audiences.</li>
<li><a class="glossary-ref" data-glossary-term="harking" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2hhcmtpbmcnIHJvbGU9J3Rvb2x0aXAnPjxwPlJlc2VhcmNoZXJzIGFyZSBIQVJLLWluZyBpZiB0aGV5IHdyaXRlIHBhcGVycyBhcyBpZiB0aGV5IGhhZCBhCmh5cG90aGVzaXMgdGhleSB3YW50ZWQgdG8gdGVzdCBpbiB0aGVpciBzdHVkeSwgd2hlcmVhcyBpbiByZWFsaXR5LCB0aGV5Cm1hZGUgdXAgdGhlIGh5cG90aGVzaXMgYWZ0ZXIgc2VlaW5nIHRoZSByZXN1bHRzLjwvcD48L2Rpdj4=" href="javascript:void(0);">HARK-ing</a>: is a shortening of ‘hypothesising after the results are known’. This is when researchers write their papers as if they had hypotheses that they then went on to test in their study, when really they made up the hypothesis after seeing their results, to pick one that best fit.</li>
<li><a class="glossary-ref" data-glossary-term="posthocjustifications" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3Bvc3Rob2NqdXN0aWZpY2F0aW9ucycgcm9sZT0ndG9vbHRpcCc+PHA+UmVzZWFyY2hlcnMgd3JpdGUgdXAganVzdGlmaWNhdGlvbnMgZm9yIHRoZWlyIGFjdGlvbnMgYWZ0ZXIgYSBzdHVkeTsKdGhlc2UganVzdGlmaWNhdGlvbnMgd2VyZSBub3QgcGxhbm5lZCBvciBkZWNpZGVkIGJlZm9yZSB0aGUgc3R1ZHkKaGFwcGVuZWQuPC9wPjwvZGl2Pg==" href="javascript:void(0);">Post-hoc justifications</a>: means stating, after the fact, justifications for decisions made during the research project. For example, if the researcher only managed to recruit women for a study after trying to recruit all genders, but claimed in the paper that this was intentional.</li>
</ul>
<p>Although pressures to publish can sometimes be seen as barriers to transparency, the benefits of writing transparently can also be seen as a positive incentive, as the next section shows.</p>
</section>
<section id="writing-transparently" class="level2" data-number="3.5">
<h2 data-number="3.5" class="anchored" data-anchor-id="writing-transparently"><span class="header-section-number">3.5</span> Writing transparently</h2>
<p>When writing manuscripts, researchers should aim to be as transparent as possible, being honest about what happened in the study, how it was conducted, and when and why decisions were made. By using questionable research practices, researchers make it more likely that they get a false positive result, which can partially explain low replicability rates.</p>
<p>In the video, Priya introduced another important consideration for evaluating replication results: sample size (the number of samples in your study, e.g. participants). Smaller sample sizes make it more likely to get both a false positive and a false negative result. This is because smaller sample sizes provide less information about the population you are studying, which increases the variability and uncertainty in your results. With a small sample, the random variation (or ‘noise’) can more easily overshadow the true effect you are trying to measure. This means you might detect an effect that isn’t really there, a false positive, or miss an effect that actually exists, a false negative.</p>
<p>For instance, imagine trying to judge the average height of a population by looking at just a few individuals. Your estimate is more likely to be off compared to measuring a larger group, because you may happen to have either a very tall or very short person in your sample. So, if you have an original study with a small sample size and a (well-designed) replication with a large sample size, you could be more confident in the result of the replication than the result of the original study.</p>
<section id="activity-2-what-not-to-do" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-2-what-not-to-do">Activity 2: What not to do!</h4>
<p><em>Allow about 10 minutes.</em></p>
<p>So far, you have considered good and bad writing practices. With these in mind, have a go at this <a href="https://stats.andrewheiss.com/hack-your-way/" target="_blank">‘hack your way to scientific glory’</a> activity. First, choose a political party: Republican or Democrat (UK equivalents: Conservative or Labour). Then predict whether the party has a positive or negative impact on the economy. When you have done that, change aspects of the research (e.g. participant inclusion criteria and how you’re measuring your dependent variable) and see whether you can find a significant result (p < 0.05) in your predicted direction.</p>
<p>The reason this is an example of ‘what not to do’ is because when you first choose a political party and predict whether they will have a positive or negative impact on the economy, you are forming a hypothesis. But, if you then play around with the data until you get the result that you wanted, and only stop when you do, then you are fixing the result.</p>
<p>The activity involves various questionable research practices, such as P-hacking, HARK-ing, selective reporting. However, there is a way to do different analyses on the same data without any of these being a problem. If instead of deciding on a hypotheses first then confirming it, you were to conduct purely exploratory research (without a hypothesis) you could be transparent about all of the different ways you looked at the data and how the results differed when you tried different things. This could even lead to people conducting their own future studies to confirm your exploratory results!</p>
<p>When reading an academic paper, it’s important to read with a critical mindset and feel free to disagree with the methodological or analysis strategy, the interpretation of the results, or the conclusions drawn. Although we know that there are rare instances of outright fraud in science, we would expect that the researchers are truthfully describing what happened in the study, how it was conducted, and when and why decisions were made.</p>
</section>
</section>
<section id="generalisability" class="level2" data-number="3.6">
<h2 data-number="3.6" class="anchored" data-anchor-id="generalisability"><span class="header-section-number">3.6</span> Generalisability</h2>
<p>You have learned that replication studies vary on a spectrum from ‘direct’ to ‘conceptual’. However, most replication studies have some differences from the original study, even if these weren’t intentional. Consider one of the examples from before, where a researcher was replicating a paper from the 1990s. The materials they create will be different from the original materials, and if what they’re studying is context-dependent, a lot might have changed since then.</p>
<p>For example, a study on internet usage habits conducted in the 1990s would yield very different results if replicated today, due to the dramatic changes in technology and how people use the internet. Similarly, a study examining public attitudes toward mental health in the 1990s might produce different findings now because societal awareness and acceptance of mental health issues have evolved significantly over the past few decades.</p>
<p>For this reason, some consider that most replication studies are actually generalisability studies. <a class="glossary-ref" data-glossary-term="generalisability" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2dlbmVyYWxpc2FiaWxpdHknIHJvbGU9J3Rvb2x0aXAnPjxwPlRoZSBleHRlbnQgdG8gd2hpY2ggdGhlIGZpbmRpbmdzIG9mIGEgc3R1ZHkgY2FuIGJlIGdlbmVyYWxpc2VkIHRvCm90aGVyIHNpdHVhdGlvbnMsIGJleW9uZCB0aGUgc3BlY2lmaWMgcGFydGljaXBhbnRzIGFuZCBjb25kaXRpb25zIG9mIHRoZQpzdHVkeS48L3A+PC9kaXY+" href="javascript:void(0);">Generalisability</a> means whether a particular result generalises beyond the specific participants and conditions of the study to broader groups of samples, settings, methods, or measures. For example, if we’re interested in public attitudes to mental health, it wouldn’t make sense for us to only ask people aged 50-60, or only men, or only those living in cities. It’s possible that any of these characteristics could affect people’s opinions on mental health, meaning the results would be biased and not representative of the full population.</p>
<p>Without generalisability studies, it might be possible that the theoretical explanation for why the finding occurred might be incorrect. For example, there could even be a mistake in the design of the study that biased the results. For instance, imagine a biological study investigating the effects of a new drug using a specific strain of lab mice. If this particular strain has a unique genetic mutation that makes it respond differently to the drug compared to other strains, the study’s results might not generalize to other mice or to humans. This could lead to an incorrect conclusion about the drug’s overall effectiveness and safety.</p>
<p>Researchers wishing to be transparent when writing their papers should declare possible ‘<a class="glossary-ref" data-glossary-term="constraintsgen" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2NvbnN0cmFpbnRzZ2VuJyByb2xlPSd0b29sdGlwJz48cD5BIHN0YXRlbWVudCBpZGVudGlmeWluZyBwb3B1bGF0aW9ucyBzYW1wbGVkIGluIHRoZSBzdHVkeSBhbmQKcG90ZW50aWFsIGxpbWl0cyB0byB0aGUgc2FtcGxlcyBhbmQgbWV0aG9kcywgZW5hYmxpbmcgb3RoZXJzIHRvIGFzc2Vzcwp0aGUgZXh0ZW50IHRvIHdoaWNoIHJlc3VsdHMgY2FuIGJlIGdlbmVyYWxpc2VkLjwvcD48L2Rpdj4=" href="javascript:void(0);">Constraints on generality</a>’ in the discussion section. This could take the form of a statement that identifies and justifies the target populations for the reported findings, and other considerations the authors think would be necessary for replicating their result. This could help other researchers to sample from the same populations when conducting a direct replication, or to test the boundaries of generalisability when conducting a conceptual replication.</p>
</section>
<section id="studying-generalisability" class="level2" data-number="3.7">
<h2 data-number="3.7" class="anchored" data-anchor-id="studying-generalisability"><span class="header-section-number">3.7</span> Studying generalisability</h2>
<p>So you think your research has potential do good in the world, but don’t know how widely it can be applied? There are lots of different ways to study generalisability:</p>
<ul>
<li><p>Systematic reviews: these look at how an outcome varies in the published literature across samples, settings, measures and methods (meta-analyses do this statistically). This can be done without conducting any new studies. For example, UNICEF’s Evidence and Gap Map Research Briefs provide an overview of available evidence of the effectiveness of interventions to improve child well-being in low- and middle-income countries.</p></li>
<li><p>Comparative studies: comparing results from different populations using the same (adapted) materials can show where there may be similarities and differences. For example, Hofstede’s cultural dimensions theory, which identified and measured cultural differences across countries, particularly in the workplace context.<br>
</p></li>
<li><p><a class="glossary-ref" data-glossary-term="bigteam" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J2JpZ3RlYW0nIHJvbGU9J3Rvb2x0aXAnPjxwPkEgcmVzZWFyY2ggcHJvamVjdCBpbiB3aGljaCByZXNlYXJjaGVycyBmcm9tIGFyb3VuZCB0aGUgd29ybGQgY29uZHVjdAp0aGUgc2FtZSBzdHVkeSBhbmQgcG9vbCB0aGVpciByZXN1bHRzLjwvcD48L2Rpdj4=" href="javascript:void(0);">Big team science</a>: when researchers from around the world conduct the same study and pool their results, they can look at various factors affecting the presence or size of the effect they’re interested in. For example, the first ManyGoats project is examining goat responses to different human attentional states, and will be testing a diverse range of goats in different living conditions.</p></li>
</ul>
<section id="activity-3" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-3">Activity 3</h4>
<p><em>Allow about 10 minutes.</em></p>
<p>Think about a study in your field that would or wouldn’t generalise. Consider why this might be the case.</p>
<p>Take some notes of your thoughts first, and see our discussion.</p>
<details>
<summary>Show / Hide Discussion </summary>
<p>
<strong>Why a study may or may not generalise</strong></p>
<p>There are lots of reasons why a study may or may not generalise. Imagine a study evaluating a new therapy for depression in a university clinic with primarily urban-based participants.</p>
<p>While the therapy showed significant improvement in depressive symptoms over ten weeks among a diverse sample, including college students and middle-aged adults of various ethnicities recruited through local health centres and university channels, its applicability to other populations and settings may be limited. Factors such as regional differences in mental health resources, demographic diversity beyond the studied age groups, and recruitment biases could affect the therapy's effectiveness in rural or suburban areas and among older adults or adolescents.</p>
</details>
</section>
</section>
<section id="quiz" class="level2" data-number="3.8">
<h2 data-number="3.8" class="anchored" data-anchor-id="quiz"><span class="header-section-number">3.8</span> Quiz</h2>
<p>This self-test quiz tackles key ideas in replication and the principle of generalisability.</p>
<div class="webex-question">
<div class="webex-check webex-box">
<ol type="1">
<li>What is the difference between replicability and reproducibility? (Select one)</li>
</ol>
<div id="webex-f67cb4f26e3cdb6e7f612fe82b4d0d96" class="webex-radiogroup" data-answer="PQYbU04FSgJr">
<label><input type="radio" autocomplete="off" name="f67cb4f26e3cdb6e7f612fe82b4d0d96"><span>There isn’t a difference.</span></label><label><input type="radio" autocomplete="off" name="f67cb4f26e3cdb6e7f612fe82b4d0d96"><span>Replicability refers to getting the same results when conducting the same study again, while reproducibility refers to getting different results when conducting the same study again.</span></label><label><input type="radio" autocomplete="off" name="f67cb4f26e3cdb6e7f612fe82b4d0d96"><span>Replicability refers to getting the same results when running the same analyses on new data, while reproducibility refers to getting the same results when conducting the same analyses on the same data.</span></label><label><input type="radio" autocomplete="off" name="f67cb4f26e3cdb6e7f612fe82b4d0d96"><span>Replicability refers to getting different results when running the same analyses on new data, while reproducibility refers to getting the same results when conducting different analyses on the same data.</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Answers</strong>:</p>
<ul>
<li>There isn’t a difference. <b>False</b></li>
<li>Replicability refers to getting the same results when conducting the same study again, while reproducibility refers to getting different results when conducting the same study again. <b>False</b></li>
<li>Replicability refers to getting the same results when running the same analyses on new data, while reproducibility refers to getting the same results when conducting the same analyses on the same data. <b>True</b></li>
<li>Replicability refers to getting different results when running the same analyses on new data, while reproducibility refers to getting the same results when conducting different analyses on the same data. <b>False</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="2" type="1">
<li>Which of the following is an example of a conceptual replication? (Select one)</li>
</ol>
<div id="webex-387219ccab3bc13f56c15f858d41e61d" class="webex-radiogroup" data-answer="aAgbAh0IT1M8">
<label><input type="radio" autocomplete="off" name="387219ccab3bc13f56c15f858d41e61d"><span>A researcher replicates a study from the 1990’s using only the methods described in the original short paper.</span></label><label><input type="radio" autocomplete="off" name="387219ccab3bc13f56c15f858d41e61d"><span>A researcher designs a study to explore how mental concepts replicate themselves.</span></label><label><input type="radio" autocomplete="off" name="387219ccab3bc13f56c15f858d41e61d"><span>A researcher designs a new study testing the same hypothesis but in a different way because they believe the original study was not well-designed.</span></label><label><input type="radio" autocomplete="off" name="387219ccab3bc13f56c15f858d41e61d"><span>A researcher conducts a replication immediately after the original study, using all the same materials and the same participant pool.</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Answers</strong>:</p>
<ul>
<li>A researcher replicates a study from the 1990’s using only the methods described in the original short paper. <b>False</b></li>
<li>A researcher designs a study to explore how mental concepts replicate themselves. <b>False</b></li>
<li>A researcher designs a new study testing the same hypothesis but in a different way because they believe the original study was not well-designed. <b>True</b></li>
<li>A researcher conducts a replication immediately after the original study, using all the same materials and the same participant pool. <b>False</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="3" type="1">
<li>Why might a replication study fail to replicate the original result? (Select one or more)</li>
</ol>
<div id="webex-4f867a06683e66695bea610af4d021cd" class="webex-checkboxgroup" data-answer="b1cUBhtQHAdr">
<label><input type="checkbox" autocomplete="off" name="4f867a06683e66695bea610af4d021cd"><span>The original result was a false positive.</span></label><label><input type="checkbox" autocomplete="off" name="4f867a06683e66695bea610af4d021cd"><span>The replication study was a waste of time.</span></label><label><input type="checkbox" autocomplete="off" name="4f867a06683e66695bea610af4d021cd"><span>The differences between the two studies are responsible for the different results.</span></label><label><input type="checkbox" autocomplete="off" name="4f867a06683e66695bea610af4d021cd"><span>The replication result was a false negative.</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: All of these are possible causes for failure to replicate. Far from being a waste of time, a failure to replicate can provide very useful information.</p>
<ul>
<li>The original result was a false positive. <b>True</b></li>
<li>The replication study was a waste of time. <b>False</b></li>
<li>The differences between the two studies are responsible for the different results. <b>True</b></li>
<li>The replication result was a false negative. <b>True</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="4" type="1">
<li>Which of the following is the best way for researchers to study generalisability? (Select one)</li>
</ol>
<div id="webex-95828740ad00fd3b1819e9508a128c72" class="webex-radiogroup" data-answer="YgQUAhQHGAA8">
<label><input type="radio" autocomplete="off" name="95828740ad00fd3b1819e9508a128c72"><span>By comparing results from different populations using the same materials.</span></label><label><input type="radio" autocomplete="off" name="95828740ad00fd3b1819e9508a128c72"><span>By re-running the original analyses from a piece of research.</span></label><label><input type="radio" autocomplete="off" name="95828740ad00fd3b1819e9508a128c72"><span>By using questionable research practices.</span></label><label><input type="radio" autocomplete="off" name="95828740ad00fd3b1819e9508a128c72"><span>By conducting direct replications with the same materials and participant pool.</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Answers</strong>:</p>
<ul>
<li>By comparing results from different populations using the same materials. <b>True</b></li>
<li>By re-running the original analyses from a piece of research. <b>False</b></li>
<li>By using questionable research practices. <b>False</b></li>
<li>By conducting direct replications with the same materials and participant pool. <b>False</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="5" type="1">
<li>Which of the following explanations are most likely if the result of an original study which suggested an effect to be ‘true’, is not replicated in a replication study? (Select one or more)</li>
</ol>
<div id="webex-a434e51790ed2f03c13e9ff2dd72b44b" class="webex-checkboxgroup" data-answer="OgUfBUkFHQdk">
<label><input type="checkbox" autocomplete="off" name="a434e51790ed2f03c13e9ff2dd72b44b"><span>The original researcher was fraudulent.</span></label><label><input type="checkbox" autocomplete="off" name="a434e51790ed2f03c13e9ff2dd72b44b"><span>The original study cannot possibly have been a ‘true’ result, because it did not replicate.</span></label><label><input type="checkbox" autocomplete="off" name="a434e51790ed2f03c13e9ff2dd72b44b"><span>The original effect might be ‘true’, but only under very narrow conditions.</span></label><label><input type="checkbox" autocomplete="off" name="a434e51790ed2f03c13e9ff2dd72b44b"><span>The replication study might have had too small a sample size to find a result.</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: The other two cannot be deduced from the information provided in the question.</p>
<ul>
<li>The original researcher was fraudulent. <b>True</b></li>
<li>The original study cannot possibly have been a ‘true’ result, because it did not replicate <b>True</b></li>
<li>The original effect might be ‘true’, but only under very narrow conditions <b>False</b></li>
<li>The replication study might have had too small a sample size to find a result <b>False</b></li>
</ul>
</div>
</div>
</section>
<section id="summary" class="level2" data-number="3.9">
<h2 data-number="3.9" class="anchored" data-anchor-id="summary"><span class="header-section-number">3.9</span> Summary</h2>
<p>In this week, you learned about an important aspect of integrity: replicability. Replicability relates to whether or not a study ‘replicates’, i.e. whether or not, when you repeat the study with new data, you get the same result. You learned some reasons why replicability may be low in many fields, and how differences between studies may sometimes contribute to this. You also learned about the importance of generalisability in research. Next week, you’ll learn techniques which can support both the integrity and the transparency of your research.</p>
</section>
</main> <!-- /main -->
<p style="text-align: center; margin: 0; padding: 0;" class="smaller">© 2025 - <a href="www.forrt.org">FORRT</a></p>
<p style="text-align: center;"><span style="color: #808080;" class="smaller"><em>Unless otherwise noted, content on this site is licensed under a <a href="https://creativecommons.org/licenses/by-sa/4.0/">CC BY NC SA 4.0 license</a></em></span></p>
<span style="text-align: center; margin: 0; padding: 0;">
<img src="assets/forrt_frane.png" width="100%">
</span>
<script>
/* definition of content for the buttons */
const webex_buttons = {check_hidden: "<b>✓</b>",
check_hidden_alt: "Check answer",
check_shown: "<b>↰</b>",
check_shown_alt: "Hide check",
check_of_total: "/",
solution: "<b>?</b>",
solution_alt: "Correct solution",
question_next: "<b>↺</b>",
question_next_alt: "Next question",
question_previous: "",
question_previous_alt: ""}
/* update total correct if #webex-total_correct exists */
update_total_correct = function() {
console.log("webex: update total_correct");
document.querySelectorAll(".webex-total_correct").forEach(total => {
p = total.closest(".webex-box");
var correct = p.getElementsByClassName("webex-correct").length;
var solvemes = p.getElementsByClassName("webex-solveme").length;
var radiogroups = p.getElementsByClassName("webex-radiogroup").length;
var selects = p.getElementsByClassName("webex-select").length;
/* no specific class on input node, thus searching via query selector */
var checkboxgroups = p.querySelectorAll("div[class=webex-checkboxgroup] input[type=checkbox]").length
/* show number of correct / total number of answers */
total.innerHTML = correct + " " + webex_buttons.check_of_total + " " + (solvemes + radiogroups + checkboxgroups + selects);
});
}
/* check answers */
check_func = function() {
console.log("webex: check answer");
//var cl = elem.parentElement.classList;
var cl = this.closest(".webex-box").classList;
if (cl.contains("unchecked")) {
cl.remove("unchecked");
this.innerHTML = webex_buttons.check_shown; //"Hide check";
if (webex_buttons.check_shown_alt.length > 0) this.setAttribute("title", webex_buttons.check_shown_alt);
} else {
cl.add("unchecked");
this.innerHTML = webex_buttons.check_hidden; //"Check answer";
if (webex_buttons.check_hidden_alt.length > 0) this.setAttribute("title", webex_buttons.check_hidden_alt);
}
}
/* helper functions for checking the webex ID */
const id_checker = {"fmt": new RegExp("[g-zG-Z]"),
"load": function(e) { return e.getAttribute("id").match("(?<=(-)).*$")[0]; },
"eval": new TextEncoder(),
"dval": new TextDecoder()}
/* Show/hide correct solution */
solution_func = function() {
console.log("webex: show/hide solution");
var div = this.closest(".webex-question").querySelector(".webex-solution");
var cl = div.classList;
if (cl.contains("visible")) {
cl.remove("visible");
} else {
cl.add("visible");
}
}
/* function to check if the real answer is numeric */
convert_to_numeric = function(x) {
if (typeof x == "string") {
/* do nothing */
} else if (x.length == 1) {
/* take first element */
x = x[0]
} else {
return NaN;
}
/* remove spaces for easier parsing */
x = x.replace(/\s/g, "");
/* Define patterns for different formats, note that spaces have been removed above */
const patterns = [
{regex: /^[+-]?\d+(\.\d{3})*,\d+$/, decimal: ",", thousand: "." }, // Format: "1.100.100.100,3"
{regex: /^[+-]?\d+(,\d{3})*\.\d+$/, decimal: ".", thousand: "," }, // Format: "1,100,100,100.3"
{regex: /^[+-]?\d+(\.|\.\d+)?$/, decimal: "." }, // Format: "1100100100.5" or "1100100100."
{regex: /^[+-]?\d+(,|,\d+)$/, decimal: "," } // Format: "1100100100,5" or "1100100100,"
];
/* testing all regular expressions, convert to float if possible */
for (const { regex, decimal, thousand } of patterns) {
if (regex.test(x)) {
let numeric = x;
if (thousand) numeric = numeric.replace(new RegExp(`\\${thousand}`, "g"), "");
numeric = numeric.replace(decimal, ".");
return parseFloat(numeric);
}
}
/* input of length 1 but none of the known formats, return NaN */
return NaN;
}
/* Checking webex ID, returns formalsol */
check_id = function(e) {
/* loading webex ID */
var id = id_checker["load"](e);
/* extracting answer */
let answer = e.dataset.answer;
/* checking unique id, prepare and return array */
if (!id_checker["fmt"].test(id)) {
const a = id_checker["eval"].encode(atob(answer));
const b = id_checker["eval"].encode(id);
const res = a.map((byte, index) => byte^b[index % b.length]);
answer = id_checker["dval"].decode(res);
/* Replacing & with ' before parsing */
answer = answer.replaceAll("'", "'");
}
return JSON.parse(answer);
}
/* function for checking solveme answers */
solveme_func = function(e) {
/* avoid that keyup and keychange twice execute this function within nms = 10
* ms, clearing inputTimer and add timeout */
console.log("webex: check solveme");
/* float, precision for checking numeric answers */
const eps = 0.00000000000001;
/* get last checked user answer */
const question = this.closest(".webex-question")
/* extracting classes */
var cl = this.classList;
var my_answer = this.value;
/* extracting classes */
var cl = this.classList;
/* empty answer? Job done */
if (my_answer == "") {
cl.remove("webex-correct");
cl.remove("webex-incorrect");
return false;
}
/* Find closest (parent) webex-question */
let formalsols = check_id(this)
/* by default we assume the users' answer is incorrect */
var user_answer_correct = false;
/* check if the correct answer is numeric, i.e. if
* formalsols is of length 1 containing one single numeric
* value in a known format, else, NaN is returned */
const num_formalsol = convert_to_numeric(formalsols);
const num_my_answer = convert_to_numeric(my_answer);
/* if the correct answer is numeric (float), the user's answer
* must also be numeric. If not, it is wrong. Else we can
* compare floating point numbers */
if (!isNaN(num_formalsol) && !isNaN(num_my_answer)) {
/* check if the real answer and the user input are numerically the same;
* adding 'delta' to avoid precision issues */
var diff = Math.abs(num_formalsol - num_my_answer);
if (diff < parseFloat(this.dataset.tol) + eps) { user_answer_correct = true; }
/* if the question contains regex, a regular expression is used
* to evaluate the users answer. Allows for multiple regular expressions */
} else if (cl.contains("regex")) {
for (let i = 0; i < formalsols.length; i++) {
let regex = new RegExp(formalsols[i], cl.contains("ignorecase") ? "i" : "");
if (regex.test(my_answer)) { user_answer_correct = true; break; }
}
/* else we evaluate on 'string level', considering the creators preferences
* regarding set options */
} else {
/* modify/prepare answer */
if (cl.contains("ignorecase")) { my_answer = my_answer.toLowerCase(); }
if (cl.contains("nospaces")) { my_answer = my_answer.replace(/ /g, ""); }
/* if the real answer includes user input - correct */
if (formalsols.includes(my_answer)) {
user_answer_correct = true;
// added regex bit
if (cl.contains("regex")) {
answer_regex = RegExp(formalsols.join("|"))
if (answer_regex.test(my_answer)) {
cl.add("webex-correct");
}
}
}
}
if (user_answer_correct) {
cl.add("webex-correct");
cl.remove("webex-incorrect");
} else {
cl.add("webex-incorrect");
cl.remove("webex-correct");
}
update_total_correct();
}
/* function for checking select answers */
select_func = function(e) {
console.log("webex: check select");
var options = [];
Array.from(this.querySelectorAll("option")).forEach((option, index) => {
if (!option.hasAttribute("value") || !option.getAttribute("value") === "blank") {
options.push(option);
}
});
/* get selected option */
function get_selected(options) { return options.findIndex(option => option.selected); }
var option_index = get_selected(options)
/* loading webex ID */
var formalsol = check_id(this)
/* add class for current selection */
this.classList.remove("webex-incorrect");
this.classList.remove("webex-correct");
if (option_index >= 0) {
this.classList.add(formalsol[option_index] === 1 ? "webex-correct" : "webex-incorrect");
}
update_total_correct();
}
/* function for checking radiogroups answers */
radiogroups_func = function(e) {
console.log("webex: check radiogroups");
/* get real answer (binary integer array) */
var id = e.target.getAttribute("name");
var group = document.querySelector(".webex-radiogroup[id='webex-" + id + "']");
var formalsol = check_id(group)
/* check which radiobuttion is selected (user answer) */
function get_checked(radios) {
return radios.findIndex(radios => radios.checked);
}
var radios = Array.from(group.querySelectorAll("input[type='radio']"))
var radio_index = get_checked(radios);
/* remove existing classes */
radios.forEach(radio => {
let cl = radio.closest("label").classList
cl.remove("webex-incorrect");
cl.remove("webex-correct");
});
/* add class for current selection */
let cl = radios[radio_index].closest("label").classList
if (formalsol[radio_index] === 1) {
cl.add("webex-correct");
} else {
cl.add("webex-incorrect");
}
update_total_correct();
}
/* function for checking checkboxgroups answers */
checkboxgroups_func = function(e) {
console.log("webex: check checkboxgroups");
/* get real answer (binary integer array) */
var group = document.querySelector(".webex-checkboxgroup[id='" + this.id + "']");
var formalsol = check_id(group);
/* check which checkbox is checked (user answer) */
function get_checked(group) {
const checkboxes = Array.from(group.querySelectorAll("input[type='checkbox']"));
return checkboxes.reduce((indices, checkbox, index) => {
if (checkbox.checked) indices.push(index);
return indices;
}, []);
}
var checks = Array.from(group.querySelectorAll("input[type='checkbox']"));
var check_index = get_checked(group);
checks.forEach((e, index) => {
var label = e.parentNode;
var ans = formalsol[index];
if ((e.checked && ans === 1) || (!e.checked && ans === 0)) {
label.setAttribute("class", "webex-correct")
} else {
label.setAttribute("class", "webex-incorrect")
}
});
update_total_correct();
}
/* shuffling array (thanks to stack overflow)
* If argument x is an integer we create an integer sequence
* from 0, 1, ..., (x - 1) and return a shuffled version. If
* the input is an array, we simply shuffle it */
shuffle_array = function(x) {
if (Number.isInteger(x) && !isNaN(x)) {
x = Array.from({length: x}, (v, i) => i);
}
let shuffled = x.map(value => ({ value, sort: Math.random() }))
.sort((a, b) => a.sort - b.sort).map(({ value }) => value)
return shuffled;
}
/* ---------------------------------------------------------
* ---------------------------------------------------------
* --------------------------------------------------------- */
window.onload = function() {
//console.log("webex: onload");
/* setting up buttons and actions to show/hide answers */
document.querySelectorAll(".webex-check").forEach(section => {
section.classList.add("unchecked");
/* ul to take up the list items with buttons */
let button_ul = document.createElement("ul");
button_ul.setAttribute("class", "webex-button-list");
section.appendChild(button_ul);
/* button to _check_ if answers given are correct */
let li_check = document.createElement("li");
button_ul.appendChild(li_check); /* add list item to ul */
let btn_check = document.createElement("button");
btn_check.innerHTML = webex_buttons.check_hidden; // "Check answer";
btn_check.setAttribute("class", "webex-button webex-button-check");
if (webex_buttons.check_hidden_alt.length > 0) btn_check.setAttribute("title", webex_buttons.check_hidden_alt);
btn_check.onclick = check_func;
li_check.appendChild(btn_check);
/* span to show current number of points (when _check_ active) */
let spn = document.createElement("span");
spn.classList.add("webex-total_correct");
li_check.appendChild(spn);
/* button to show the _solution_ if there is one */
var has_solution = section.parentNode.querySelectorAll(".webex-solution").length > 0;
if (has_solution) {
let li_solution = document.createElement("li");
button_ul.appendChild(li_solution); /* add list item to ul */
let btn_solution = document.createElement("button");
btn_solution.innerHTML = webex_buttons.solution; // "Correct answer";
btn_solution.setAttribute("class", "webex-button webex-button-solution");
if (webex_buttons.solution_alt.length > 0) btn_solution.setAttribute("title", webex_buttons.solution_alt);
btn_solution.onclick = solution_func;
li_solution.appendChild(btn_solution);
}
});
/* set up webex-solveme inputs */
document.querySelectorAll(".webex-solveme").forEach(solveme => {
solveme.setAttribute("autocomplete","off");
solveme.setAttribute("autocorrect", "off");
solveme.setAttribute("autocapitalize", "off");
solveme.setAttribute("spellcheck", "false");
solveme.value = "";
/* adjust answer for 'no spaces' (ignore spaces) */
if (solveme.classList.contains("nospaces")) {
solveme.dataset.answer = solveme.dataset.answer.replace(/ /g, "");
}