-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathWeek5.html
More file actions
1706 lines (1562 loc) · 94.6 KB
/
Week5.html
File metadata and controls
1706 lines (1562 loc) · 94.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
<meta charset="utf-8">
<meta name="generator" content="quarto-1.7.32">
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
<title>5 Integrity: Supporting robust interpretations – Open Research</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
width: 0.8em;
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
vertical-align: middle;
}
</style>
<script src="site_libs/quarto-nav/quarto-nav.js"></script>
<script src="site_libs/quarto-nav/headroom.min.js"></script>
<script src="site_libs/clipboard/clipboard.min.js"></script>
<script src="site_libs/quarto-search/autocomplete.umd.js"></script>
<script src="site_libs/quarto-search/fuse.min.js"></script>
<script src="site_libs/quarto-search/quarto-search.js"></script>
<meta name="quarto:offset" content="./">
<link href="./Week6.html" rel="next">
<link href="./Week4.html" rel="prev">
<script src="site_libs/quarto-html/quarto.js" type="module"></script>
<script src="site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
<script src="site_libs/quarto-html/popper.min.js"></script>
<script src="site_libs/quarto-html/tippy.umd.min.js"></script>
<script src="site_libs/quarto-html/anchor.min.js"></script>
<link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
<link href="site_libs/quarto-html/quarto-syntax-highlighting-37eea08aefeeee20ff55810ff984fec1.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="site_libs/bootstrap/bootstrap.min.js"></script>
<link href="site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="site_libs/bootstrap/bootstrap-7e4aeafd2f2715da386b171fb548fac0.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
<link href="site_libs/quarto-contrib/popover-glossary/popover-glossary.css" rel="stylesheet">
<script id="quarto-search-options" type="application/json">{
"location": "sidebar",
"copy-button": false,
"collapse-after": 3,
"panel-placement": "start",
"type": "textbox",
"limit": 50,
"keyboard-shortcut": [
"f",
"/",
"s"
],
"show-item-context": false,
"language": {
"search-no-results-text": "No results",
"search-matching-documents-text": "matching documents",
"search-copy-link-title": "Copy link to search",
"search-hide-matches-text": "Hide additional matches",
"search-more-match-text": "more match in this document",
"search-more-matches-text": "more matches in this document",
"search-clear-button-title": "Clear",
"search-text-placeholder": "",
"search-detached-cancel-button-title": "Cancel",
"search-submit-button-title": "Submit",
"search-label": "Search"
}
}</script>
<link rel="shortcut icon" href="assets/forrt_icon.png">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css">
<style>
div.callout-green.callout {
border-left-color: #bad6c2;
}
div.callout-green.callout-style-default > .callout-header {
background-color: rgba(186, 214, 194, 0.13);
}
div.callout-green .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-green.callout-style-default .callout-icon::before, div.callout-green.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f0eb' !important;
background-image: none;
}
div.callout-caution.callout {
border-left-color: #ff9a00;
}
div.callout-caution.callout-style-default > .callout-header {
background-color: rgba(255, 154, 0, 0.13);
}
div.callout-caution .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-caution.callout-style-default .callout-icon::before, div.callout-caution.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f071' !important;
background-image: none;
}
div.callout-red.callout {
border-left-color: #8e0000;
}
div.callout-red.callout-style-default > .callout-header {
background-color: rgba(142, 0, 0, 0.13);
}
div.callout-red .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-red.callout-style-default .callout-icon::before, div.callout-red.callout-titled .callout-icon::before {
background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="%238e0000" class="bi bi-exclamation-triangle" viewBox="0 0 16 16"><path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/></svg>');
}
div.callout-yellow.callout {
border-left-color: #86754a;
}
div.callout-yellow.callout-style-default > .callout-header {
background-color: rgba(134, 117, 74, 0.13);
}
div.callout-yellow .callout-toggle::before { background-image: url('data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="rgb(33, 37, 41)" class="bi bi-chevron-down" viewBox="0 0 16 16"><path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/></svg>');}
div.callout-yellow.callout-style-default .callout-icon::before, div.callout-yellow.callout-titled .callout-icon::before {
font-family: 'Font Awesome 6 Free', FontAwesome;
font-style: normal;
content: '\f05a' !important;
background-image: none;
}
</style>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="include/webex.css">
</head>
<body class="nav-sidebar floating quarto-light">
<div id="quarto-search-results"></div>
<header id="quarto-header" class="headroom fixed-top">
<nav class="quarto-secondary-nav">
<div class="container-fluid d-flex">
<button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
<i class="bi bi-layout-text-sidebar-reverse"></i>
</button>
<nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./Week1.html">Chapters</a></li><li class="breadcrumb-item"><a href="./Week5.html"><span class="chapter-number">5</span> <span class="chapter-title">Integrity: Supporting robust interpretations</span></a></li></ol></nav>
<a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
</a>
<button type="button" class="btn quarto-search-button" aria-label="Search" onclick="window.quartoOpenSearch();">
<i class="bi bi-search"></i>
</button>
</div>
</nav>
</header>
<!-- content -->
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-full">
<!-- sidebar -->
<nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation floating overflow-auto">
<div class="pt-lg-2 mt-2 text-left sidebar-header sidebar-header-stacked">
<a href="./index.html" class="sidebar-logo-link">
<img src="https://forrt.org/img/FORRT.svg" alt="" class="sidebar-logo py-0 d-lg-inline d-none">
</a>
<div class="sidebar-title mb-0 py-0">
<a href="./">Open Research</a>
<div class="sidebar-tools-main">
<a href="https://github.com/forrtproject/Open-Research-course" title="Source Code" class="quarto-navigation-tool px-1" aria-label="Source Code"><i class="bi bi-github"></i></a>
</div>
</div>
</div>
<div class="mt-2 flex-shrink-0 align-items-center">
<div class="sidebar-search">
<div id="quarto-search" class="" title="Search"></div>
</div>
</div>
<div class="sidebar-menu-container">
<ul class="list-unstyled mt-1">
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
<span class="menu-text">Overview</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./index.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Overview</span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
<span class="menu-text">Chapters</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week1.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">1</span> <span class="chapter-title">What is Open Research</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week2.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">2</span> <span class="chapter-title">Transparency: As open as possible</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week3.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">3</span> <span class="chapter-title">Integrity: Challenging questionable research practices</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week4.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">4</span> <span class="chapter-title">Documenting Decisions Transparently</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week5.html" class="sidebar-item-text sidebar-link active">
<span class="menu-text"><span class="chapter-number">5</span> <span class="chapter-title">Integrity: Supporting robust interpretations</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week6.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">6</span> <span class="chapter-title">Accessibility: Making your research accessible online</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week7.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">7</span> <span class="chapter-title">Accessibility: Academic privilege and diversity</span></span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./Week8.html" class="sidebar-item-text sidebar-link">
<span class="menu-text"><span class="chapter-number">8</span> <span class="chapter-title">Committing to open research</span></span></a>
</div>
</li>
</ul>
</li>
<li class="sidebar-item sidebar-item-section">
<div class="sidebar-item-container">
<a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true">
<span class="menu-text">Appendices</span></a>
<a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-3" role="navigation" aria-expanded="true" aria-label="Toggle section">
<i class="bi bi-chevron-right ms-2"></i>
</a>
</div>
<ul id="quarto-sidebar-section-3" class="collapse list-unstyled sidebar-section depth1 show">
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./glossary.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Glossary</span></a>
</div>
</li>
<li class="sidebar-item">
<div class="sidebar-item-container">
<a href="./reference.html" class="sidebar-item-text sidebar-link">
<span class="menu-text">Reference</span></a>
</div>
</li>
</ul>
</li>
</ul>
</div>
</nav>
<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
<!-- margin-sidebar -->
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
<nav id="TOC" role="doc-toc" class="toc-active">
<h2 id="toc-title">Table of contents</h2>
<ul>
<li><a href="#conflicts-of-interest" id="toc-conflicts-of-interest" class="nav-link active" data-scroll-target="#conflicts-of-interest"><span class="header-section-number">5.1</span> Conflicts of interest</a></li>
<li><a href="#positionality" id="toc-positionality" class="nav-link" data-scroll-target="#positionality"><span class="header-section-number">5.2</span> Positionality</a>
<ul>
<li><a href="#positionality-statements" id="toc-positionality-statements" class="nav-link" data-scroll-target="#positionality-statements"><span class="header-section-number">5.2.1</span> Positionality statements</a>
<ul class="collapse">
<li><a href="#activity-1" id="toc-activity-1" class="nav-link" data-scroll-target="#activity-1">Activity 1</a></li>
</ul></li>
</ul></li>
<li><a href="#flimsy-interpretations" id="toc-flimsy-interpretations" class="nav-link" data-scroll-target="#flimsy-interpretations"><span class="header-section-number">5.3</span> Flimsy interpretations?</a>
<ul>
<li><a href="#avoiding-the-pitfalls" id="toc-avoiding-the-pitfalls" class="nav-link" data-scroll-target="#avoiding-the-pitfalls"><span class="header-section-number">5.3.1</span> Avoiding the pitfalls</a>
<ul class="collapse">
<li><a href="#activity-2" id="toc-activity-2" class="nav-link" data-scroll-target="#activity-2">Activity 2</a></li>
</ul></li>
</ul></li>
<li><a href="#robustness" id="toc-robustness" class="nav-link" data-scroll-target="#robustness"><span class="header-section-number">5.4</span> Robustness</a>
<ul>
<li><a href="#multiverse-analysis" id="toc-multiverse-analysis" class="nav-link" data-scroll-target="#multiverse-analysis"><span class="header-section-number">5.4.1</span> Multiverse analysis</a>
<ul class="collapse">
<li><a href="#activity-3" id="toc-activity-3" class="nav-link" data-scroll-target="#activity-3">Activity 3</a></li>
</ul></li>
</ul></li>
<li><a href="#the-open-research-decision-tree" id="toc-the-open-research-decision-tree" class="nav-link" data-scroll-target="#the-open-research-decision-tree"><span class="header-section-number">5.5</span> The Open Research Decision Tree</a></li>
<li><a href="#quiz" id="toc-quiz" class="nav-link" data-scroll-target="#quiz"><span class="header-section-number">5.6</span> Quiz</a></li>
<li><a href="#summary" id="toc-summary" class="nav-link" data-scroll-target="#summary"><span class="header-section-number">5.7</span> Summary</a></li>
</ul>
<div class="toc-actions"><ul><li><a href="https://github.com/forrtproject/Open-Research-course/issues" class="toc-action"><i class="bi bi-github"></i>Report an issue</a></li><li><a href="https://github.com/forrtproject/Open-Research-course/edit/main/Week5.qmd" class="toc-action"><i class="bi empty"></i>Edit this page</a></li></ul></div></nav>
</div>
<!-- main -->
<main class="content column-body" id="quarto-document-content">
<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./Week1.html">Chapters</a></li><li class="breadcrumb-item"><a href="./Week5.html"><span class="chapter-number">5</span> <span class="chapter-title">Integrity: Supporting robust interpretations</span></a></li></ol></nav>
<div class="quarto-title">
<h1 class="title"><span class="chapter-number">5</span> <span class="chapter-title">Integrity: Supporting robust interpretations</span></h1>
</div>
<div class="quarto-title-meta column-body">
</div>
</header>
<p>Researchers are human, and humans have their own experiences, perspectives, values, and biases. For this reason, we must be alert to the possibility that any individual researcher’s observations could be influenced by their position.</p>
<p>In Week 3, you looked at the principle of integrity in open research: you will finish your consideration of integrity in Week 5. You will learn how to protect yourself from unintended bias, spot the signs of flimsy analysis, and scrutinise the robustness of your own research. By the end of this week, you will have a toolkit of techniques which you can use to support the integrity of your research.</p>
<section id="conflicts-of-interest" class="level2" data-number="5.1">
<h2 data-number="5.1" class="anchored" data-anchor-id="conflicts-of-interest"><span class="header-section-number">5.1</span> Conflicts of interest</h2>
<p>It is standard in all types of research to disclose any specific ‘conflicts of interest’ – factors that could make the researcher biased towards particular results. This is common in research projects where money is involved. Would the researcher (or the company funding the researcher) benefit from the results of the research coming out in a particular direction?</p>
<p>Here are some examples of potential conflicts of interest:</p>
<ul>
<li><p>A pharmaceutical company funding a clinical trial for its own drug may create a conflict of interest, if the company stands to benefit financially from positive results.</p></li>
<li><p>A researcher receiving funding from an oil company to study the environmental impacts of drilling may face conflicts of interest, if they are under pressure to downplay negative findings.</p></li>
<li><p>Researchers studying the effectiveness of educational interventions funded by the companies producing those interventions may have conflicts of interest if their findings recommend using the products of those companies.</p></li>
</ul>
<p>While it is common to disclose conflicts of interest, there are many fields where it isn’t common to consider how our experiences, perspectives, values, and biases might affect our research in a way that is less clear-cut.</p>
</section>
<section id="positionality" class="level2" data-number="5.2">
<h2 data-number="5.2" class="anchored" data-anchor-id="positionality"><span class="header-section-number">5.2</span> Positionality</h2>
<p><a class="glossary-ref" data-glossary-term="positionality" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3Bvc2l0aW9uYWxpdHknIHJvbGU9J3Rvb2x0aXAnPjxwPlJlZmVycyB0byB0aGUgc29jaWFsIGFuZCBwb2xpdGljYWwgcG9zaXRpb24gb2YgYW4gaW5kaXZpZHVhbCB3aXRoaW4Kc29jaWV0eSwgaW5jbHVkaW5nIHRoZWlyIGlkZW50aXR5LCBiYWNrZ3JvdW5kLCBleHBlcmllbmNlcywgYW5kCmJlbGllZnMuPC9wPjwvZGl2Pg==" href="javascript:void(0);">Positionality</a> refers to an individual’s social and political position within society, including their identity, background, experiences, and beliefs. These factors can influence the way researchers perceive and interpret data, potentially impacting the research process and outcomes. Positionality is not always negative – a researcher can also be uniquely positioned to study something because they have a deeper lived understanding of it.</p>
<p>Here are some positive and negative examples of ways that positionality could influence research:</p>
<ul>
<li><p>Gender biases among medical professionals may affect the way certain health conditions are studied or treated.</p></li>
<li><p>Economic researchers’ ideological beliefs and political affiliations can shape their interpretations of data and policy recommendations.</p></li>
<li><p>Indigenous researchers may offer traditional ecological knowledge that complements non-indigenous approaches, leading to innovative conservation strategies.</p></li>
<li><p>An activist-researcher may use their lived experience of living under a dictatorship when conducting a study on political systems.</p></li>
</ul>
<section id="positionality-statements" class="level3" data-number="5.2.1">
<h3 data-number="5.2.1" class="anchored" data-anchor-id="positionality-statements"><span class="header-section-number">5.2.1</span> Positionality statements</h3>
<p>Positionality statements allow readers to assess the positionality of a researcher, and how it might affect their research. They are common in qualitative research, and are starting to be considered in quantitative research, too.</p>
<p>Researchers can consider including a ‘positionality statement’ in papers, to contextualise themselves and their research environment, and define the boundaries of their research output. This can provide additional context around how the study was conducted, including the researcher’s experiences, perspectives, and potential biases.</p>
<p>Positionality statements and conflict of interest statements both serve to disclose personal biases or influences that might affect an individual’s work or perspective. However, they differ in scope and intent. Positionality statements focus more heavily on the author’s social and cultural identity factors, such as race, gender, and socioeconomic status, to provide context on how these factors might influence their viewpoint. Conflict of interest statements, on the other hand, disclose financial or personal relationships that could compromise the integrity of one’s work or create a perception of bias. While both aim for transparency, positionality statements address broader socio-cultural influences, whereas conflict of interest statements specifically target potential financial or relational biases.</p>
<p><a class="glossary-ref" data-glossary-term="qualitative" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3F1YWxpdGF0aXZlJyByb2xlPSd0b29sdGlwJz48cD4oRnJvbSBGT1JSVCBHbG9zc2FyeSkgUmVzZWFyY2ggd2hpY2ggdXNlcyBub24tbnVtZXJpY2FsIGRhdGEsIHN1Y2ggYXMKdGV4dHVhbCByZXNwb25zZXMsIGltYWdlcywgdmlkZW9zIG9yIG90aGVyIGFydGVmYWN0cywgdG8gZXhwbG9yZQppbi1kZXB0aCBjb25jZXB0cywgdGhlb3JpZXMsIG9yIGV4cGVyaWVuY2VzLiBUaGVyZSBhcmUgYSB3aWRlIHJhbmdlIG9mCnF1YWxpdGF0aXZlIGFwcHJvYWNoZXMsIGZyb20gbWljcm8tZGV0YWlsZWQgZXhwbG9yYXRpb24gb2YgbGFuZ3VhZ2Ugb3IKZm9jdXNpbmcgb24gcGVyc29uYWwgc3ViamVjdGl2ZSBleHBlcmllbmNlcywgdG8gdGhvc2Ugd2hpY2ggZXhwbG9yZQptYWNyby1sZXZlbCBzb2NpYWwgZXhwZXJpZW5jZXMgYW5kIG9waW5pb25zLjwvcD48L2Rpdj4=" href="javascript:void(0);">Qualitative researchers</a> often talk about <a class="glossary-ref" data-glossary-term="reflex1" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3JlZmxleDEnIHJvbGU9J3Rvb2x0aXAnPjxwPihGcm9tIEZPUlJUIEdsb3NzYXJ5KSBUaGUgcHJvY2VzcyBvZiByZWZsZXhpdml0eSByZWZlcnMgdG8gY3JpdGljYWxseQpjb25zaWRlcmluZyB0aGUga25vd2xlZGdlIHRoYXQgd2UgcHJvZHVjZSB0aHJvdWdoIHJlc2VhcmNoLCBob3cgaXQgaXMKcHJvZHVjZWQsIGFuZCBvdXIgb3duIHJvbGUgYXMgcmVzZWFyY2hlcnMgaW4gcHJvZHVjaW5nIHRoaXMga25vd2xlZGdlLgpUaGVyZSBhcmUgZGlmZmVyZW50IGZvcm1zIG9mIHJlZmxleGl2aXR5OyBwZXJzb25hbCByZWZsZXhpdml0eSB3aGVyZWJ5CnJlc2VhcmNoZXJzIGNvbnNpZGVyIHRoZSBpbXBhY3Qgb2YgdGhlaXIgb3duIHBlcnNvbmFsIGV4cGVyaWVuY2VzLCBhbmQKZnVuY3Rpb25hbCB3aGVyZWJ5IHJlc2VhcmNoZXJzIGNvbnNpZGVyIHRoZSB3YXkgaW4gd2hpY2ggb3VyIHJlc2VhcmNoCnRvb2xzIGFuZCBtZXRob2RzIG1heSBoYXZlIGltcGFjdGVkIGtub3dsZWRnZSBwcm9kdWN0aW9uLiBSZWZsZXhpdml0eQphaW1zIHRvIGJyaW5nIGF0dGVudGlvbiB0byB1bmRlcmx5aW5nIGZhY3RvcnMgd2hpY2ggbWF5IGltcGFjdCB0aGUKcmVzZWFyY2ggcHJvY2VzcywgaW5jbHVkaW5nIGRldmVsb3BtZW50IG9mIHJlc2VhcmNoIHF1ZXN0aW9ucywgZGF0YQpjb2xsZWN0aW9uLCBhbmQgdGhlIGFuYWx5c2lzLjwvcD48L2Rpdj4=" href="javascript:void(0);">reflexivity</a> – a researcher’s ability to reflect critically on their own position, and how it influences the research process. This is a key skill in the social sciences and other disciplines that use qualitative research methods for studying social interaction, interpersonal relations, or cultural practices. A dash of reflexivity is particularly helpful for writing a positionality statement: see <a href="https://medium.com/@Marvette/just-tell-me-what-i-need-to-know-reflexivity-and-positionality-statements-fb52ec0f4e17" target="_blank">this article</a> for more on reflexivity and positionality statements.</p>
<section id="activity-1" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-1">Activity 1</h4>
<p><em>Allow about 30 minutes</em></p>
<p>Write a positionality statement for yourself. You can do this for a research project you are involved in, one you have been involved in in the past, or for a fictional project in a field you are interested in.</p>
<p>As you do so, think about whether your positionality might influence the way you perceive and interpret the data in the project you have chosen, and whether it might impact the research process and outcomes in both negative and positive ways. For instance, your negative experience with police after being a witness to a crime might influence your interpretation of police data relating to lineup processes in a way that does not reflect reality. In contrast, your experience of having a child with a rare disease might help to ask the right questions when interviewing other parents with similar experiences.</p>
<p><img src="images/writing-icon-credit-SmashIcons.png" class="img-fluid quarto-figure quarto-figure-left" width="50"> Write your positionality statement in your notes.</p>
</section>
</section>
</section>
<section id="flimsy-interpretations" class="level2" data-number="5.3">
<h2 data-number="5.3" class="anchored" data-anchor-id="flimsy-interpretations"><span class="header-section-number">5.3</span> Flimsy interpretations?</h2>
<p>In Week 3, you learned how researchers can be biased towards statistically significant results, and results that fit with the story they are trying to tell in their paper. One of the ways to spot this is when results and conclusions don’t follow on from each other. Tenuous links between results and conclusions are not always obvious, but they will be easier to spot once you are familiar with papers in your particular research area. Here are some to avoid in your own research:</p>
<div class="callout callout-style-default callout-caution callout-titled">
<div class="callout-header d-flex align-content-center">
<div class="callout-icon-container">
<i class="callout-icon"></i>
</div>
<div class="callout-title-container flex-fill">
P-value interpretation
</div>
</div>
<div class="callout-body-container callout-body">
<p>In quantitative papers where a specific {{<term “pvalue|p-value”>}} threshold is being used to determine whether a result is statistically significant or not, researchers should specify at the beginning of their analysis section what their threshold will be for interpreting significance (e.g. < 0.05). It’s important that p-values are interpreted consistently throughout the analysis. For example, you shouldn’t find 0.05 being used to show a significant difference in one case, but not in another. Any statistical value that is larger than your identified threshold should not be presented as evidence of an effect or association, however much you may wish it to be so!</p>
</div>
</div>
<div class="callout callout-style-default callout-caution callout-titled">
<div class="callout-header d-flex align-content-center">
<div class="callout-icon-container">
<i class="callout-icon"></i>
</div>
<div class="callout-title-container flex-fill">
Support for theory
</div>
</div>
<div class="callout-body-container callout-body">
<p>Sometimes, researchers can be so invested in a particular theory they are not able to see other ways their results could be interpreted. You should always try to think about alternative explanations for your results, and include these in the manuscript discussion. When reading other researchers’ papers, think about other possible interpretations of their results, and evidence for and against those different interpretations. It can be difficult to see theories outside your own position, so it is helpful to get other researchers with different experiences or expertise to read your work before submitting it. You can offer to do the same for them when they are writing a manuscript.</p>
</div>
</div>
<div class="callout callout-style-default callout-caution callout-titled">
<div class="callout-header d-flex align-content-center">
<div class="callout-icon-container">
<i class="callout-icon"></i>
</div>
<div class="callout-title-container flex-fill">
Burying results
</div>
</div>
<div class="callout-body-container callout-body">
<p>Sometimes researchers present several results in an article, but ‘cherry-pick’ which of these to highlight in the discussion, overemphasising results that fit the story they’re trying to tell in their paper, and underemphasising those that seem to be contradictory. It’s important that any contradictory results are included in the discussion section, with speculation about why they may have occurred.</p>
</div>
</div>
<p>In Week 3, we pointed out that these biases are largely due to problematic incentive structures in academia. Researchers are incentivised to publish exciting, significant results in their papers, as these are more easily accepted by highly-regarded journals. Knowing this, it isn’t surprising that researchers are often biased to tell a simple, effective story in their papers, even though research is messy!</p>
<p>Slowly, the norms do seem to be shifting, so it is becoming more common to be fully transparent in your manuscript writing, by including potentially confusing results and being honest about uncertainties.</p>
<section id="avoiding-the-pitfalls" class="level3" data-number="5.3.1">
<h3 data-number="5.3.1" class="anchored" data-anchor-id="avoiding-the-pitfalls"><span class="header-section-number">5.3.1</span> Avoiding the pitfalls</h3>
<p>Sometimes academic journals have specific word counts, and it can be difficult to fit a lot of nuance and several additional analyses into the body of the paper. Where this is the case, it can be helpful to include all this information as supplementary information accompanying the paper (e.g. as a document uploaded to the Open Science Framework) rather than in the paper itself.</p>
<p>Preregistration, which you learned about in Week 4, can also help you to avoid the pitfalls described on the previous page. If you plan to use p-values to make conclusions about whether your statistical tests are significant or not, you will need to outline a significance threshold in your preregistration. You will also need to outline how you will interpret different results, including whether they will support a specific theory.</p>
<p>Preregistration can also protect you from burying results that don’t support your conclusions: if you preregister that you will run certain analyses, you will need to report the results of these, regardless of what they were.</p>
<p>Importantly, without anyone checking your preregistration, you could still make decisions in the preregistration that make your results less credible. For example, you could pick a very high significance threshold (e.g. p < 0.1), which would make false positive results more likely. You could say that a result supports a theory that doesn’t make sense, or you could ‘bury’ additional analyses that negate the results you’d prefer to draw attention to, if they weren’t included in your preregistration. Preregistration doesn’t automatically give your work more integrity, but it can help you to think through your research decisions more clearly before you start, and stop you from tricking yourself later.</p>
<section id="activity-2" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-2">Activity 2</h4>
<p><em>Allow about 30 minutes</em></p>
<p>Think of a disagreement you’ve had with someone. Write down three versions of the disagreement: one where you’re completely right, one where the other person is completely right, and one where you explain the complicated truth!</p>
<div class="quarto-figure quarto-figure-left">
<figure class="figure">
<p><img src="images/writing-icon-credit-SmashIcons.png" class="img-fluid quarto-figure quarto-figure-left figure-img" width="50"></p>
</figure>
</div>
<details>
<summary>Show / Hide Discussion</summary>
<p>
<br>
Reflect on the activity – was it more difficult to write someone else’s perspective rather than your own? How might this manifest in research? Might it be easier to write about how your results support your preferred theory than to think about alternatives? Could stepping into the shoes of another researcher help you to try to work out alternative explanations for your results?
</p>
</details>
</section>
</section>
</section>
<section id="robustness" class="level2" data-number="5.4">
<h2 data-number="5.4" class="anchored" data-anchor-id="robustness"><span class="header-section-number">5.4</span> Robustness</h2>
<p><a class="glossary-ref" data-glossary-term="robustness" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J3JvYnVzdG5lc3MnIHJvbGU9J3Rvb2x0aXAnPjxwPlJlZmVycyB0byB0aGUgc3RyZW5ndGggYW5kIHJlbGlhYmlsaXR5IG9mIHJlc3VsdHMuPC9wPjwvZGl2Pg==" href="javascript:void(0);">Robustness</a> refers to the strength and reliability of results. Results can be considered more robust (and therefore have more integrity) if they hold up under various conditions, e.g. different data analyses. When results are robust to different data analyses, this indicates that the conclusions drawn from the research are not overly dependent on specific features of one type of analysis, and are therefore likely more widely applicable.</p>
<div class="quarto-figure quarto-figure-center">
<figure class="figure">
<p><img src="https://www.open.edu/openlearncreate/pluginfile.php/829732/mod_oucontent/oucontent/87237/9d2c3b0a/a52a109b/bridges.jpg" class="img-fluid figure-img"></p>
<figcaption>A ruined wooden bridge on the left and a cast iron bridge on the right. Let’s check the integrity of the bridges here. The bridge on the left is likely to be less robust to extreme weather events than the bridge on the right. Which would you rather cross?</figcaption>
</figure>
</div>
<p>In some fields, it’s common to run many robustness analyses. For example, in economics, it is typical to have dozens of pages of any paper showing that a particular result holds up no matter how you measure the variables, which participants you include, which statistical model you use, and even when you control for a variety of factors that could be an alternative explanation for the effect.</p>
<p>However, in other fields it’s less common. For example, in psychology, papers are often published where only one key analysis is performed to examine the results. If data and materials aren’t shared openly, it means others outside of the original research team cannot even choose to run these analyses themselves to check the robustness of the results. This is a good example of how integrity is difficult to check without transparency.</p>
<section id="multiverse-analysis" class="level3" data-number="5.4.1">
<h3 data-number="5.4.1" class="anchored" data-anchor-id="multiverse-analysis"><span class="header-section-number">5.4.1</span> Multiverse analysis</h3>
<p>One way to take robustness to its most extreme is to perform <a class="glossary-ref" data-glossary-term="multiverseanalysis" data-glossary-def-base64="PGRpdiBjbGFzcz0nZ2xvc3NhcnktZGVmJyBkYXRhLWdsb3NzYXJ5LXRlcm09J211bHRpdmVyc2VhbmFseXNpcycgcm9sZT0ndG9vbHRpcCc+PHA+U3lzdGVtYXRpY2FsbHkgc2FtcGxpbmcgYSB2YXN0IHNldCBvZiBzcGVjaWZpY2F0aW9ucywga25vd24gYXMgYQptdWx0aXZlcnNlLCB0byBlc3RpbWF0ZSB0aGUgdW5jZXJ0YWludHkgc3Vycm91bmRpbmcgdGhlIHZhbGlkaXR5IG9mIGEKc2NpZW50aWZpYyBjbGFpbS48L3A+PC9kaXY+" href="javascript:void(0);">multiverse analysis</a>. Although this sounds like something out of a science fiction movie about time travel - it’s actually a lot less out-of-this-world (however, still very cool). Multiverse analysis is where researchers try to perform all possible reasonable analyses on the data, in order to explore which analyses show the effect they’re interested in and which don’t.</p>
<section id="activity-3" class="level4 unnumbered">
<h4 class="unnumbered anchored" data-anchor-id="activity-3">Activity 3</h4>
<p><em>Allow 10 minutes.</em></p>
<p>Using the <a href="https://stats.andrewheiss.com/hack-your-way/" target="_blank">hack your way to scientific glory</a> activity from <a href="https://forrt.org/" target="_blank">Week 3</a>, try to write down as many different combinations of analysis parameters and the results you get from them (perform and write down as many as you can in 10 minutes). In effect, you will be doing a mini-multiverse analysis. What does it mean that the results are so variable, depending on the analysis performed?</p>
<div class="quarto-figure quarto-figure-left">
<figure class="figure">
<p><img src="images/writing-icon-credit-SmashIcons.png" class="img-fluid quarto-figure quarto-figure-left figure-img" width="50"></p>
</figure>
</div>
<details>
<summary>Show / Hide Discussion</summary>
<p>
<br>
There are actually NINE HUNDRED different analysis combinations you could use in this activity! Conducting a full multiverse analysis on data like these would take a very long time, and go beyond the scope of the usual research project. But perhaps thinking about some of the most important robustness checks you could do with your research could be a first step towards enhanced robustness, without exploring the entire multiverse.
</p>
</details>
</section>
</section>
</section>
<section id="the-open-research-decision-tree" class="level2" data-number="5.5">
<h2 data-number="5.5" class="anchored" data-anchor-id="the-open-research-decision-tree"><span class="header-section-number">5.5</span> The Open Research Decision Tree</h2>
<p><em>Allow 10 minutes for this activity.</em><br>
<strong>Click on different parts of the image to go to the relevant section within this course.</strong></p>
<img src="images/decision_tree.png" alt="Open science decision tree" usemap="#workmap" width="100%">
<map name="workmap">
<area target="_blank" alt="Preregistration: publishing your plans for a study" title="Preregistration: publishing your plans for a study" href="https://forrt.org/open-research-course/Week4.html#preregistration-publishing-your-plans-for-a-study" coords="301,779,51,549" shape="rect">
<area shape="rect" coords="300,942,50,788" alt="2. Open Data" href="https://forrt.org/open-research-course/Week2.html#open-data" target="_blank">
<area shape="rect" coords="300,1160,50,946" alt="3. Where to Share Data" href="https://forrt.org/open-research-course/Week2.html#where-to-share-data">
<area shape="rect" coords="600,774,352,545" alt="4. The replication crisis" href="https://forrt.org/open-research-course/Week3.html#the-replication-crisis " target="_blank">
<area shape="rect" coords="600,942,352,788" alt="5. Generalisability" href="https://forrt.org/open-research-course/Week3.html#generalisability" target="_blank">
<area shape="rect" coords="600,1160,352,946" alt="6. Positionality" href="https://forrt.org/open-research-course/Week5.html#positionality" target="_blank">
<area shape="rect" coords="906,780,650,545" alt="7. Open access" href="https://forrt.org/open-research-course/Week6.html#open-access" target="_blank">
<area shape="rect" coords="650,788,906,942" alt="8. Preprints" href="https://forrt.org/open-research-course/Week6.html#preprints" target="_blank">
<area shape="rect" coords="906,1160,650,946" alt="9. Privilege in academia" href="https://forrt.org/open-research-course/Week7.html#privilege-in-academia" target="_blank">
<area shape="rect" coords="1310, 780, 1058, 545" alt="10. The FAIR principles" href="https://forrt.org/open-research-course/Week2.html#the-fair-principles" target="_blank">
<area shape="rect" coords="1310,942,1058,788" alt="11. Robustness" href="https://forrt.org/open-research-course/Week5.html#robustness" target="_blank">
<area shape="rect" coords="1310,1160, 1058,946" alt="12. Big team science" href="https://forrt.org/open-research-course/Week7.html#big-team-science" target="_blank">
<area shape="rect" coords="1608,780,1358,545" alt="13. Open access repositories" href="https://forrt.org/open-research-course/Week8.html#open-access-repositories" target="_blank">
<area shape="rect" coords="1608,942,1358,788" alt="14. Replicability" href="https://forrt.org/open-research-course/Week3.html#replicability" target="_blank">
<area shape="rect" coords="1608,1160,1358,946" alt="15. Finding your community" href="https://forrt.org/open-research-course/Week8.html#finding-your-community" target="_blank">
<area shape="rect" coords="1900, 780,1600, 545" alt="16. Journal publishing models" href="https://forrt.org/open-research-course/Week6.html#journal-publishing-models" target="_blank">
<area shape="rect" coords="1900,942,1600,788" alt="17. Preprints" href="https://forrt.org/open-research-course/Week6.html#preprints" target="_blank">
<area shape="rect" coords="1900,1160,1660,946" alt="18. Open Access" href="https://forrt.org/open-research-course/Week6.html#open-access" target="_blank">
</map>
<script src="include/ImageMapResizer.min.js"></script>
<script>
imageMapResize();
</script>
<p>Here’s another opportunity to explore the open research decision tree. Consider ‘Actions’. The ‘Actions’ pathway suggests concrete steps you can take at three different stages: the planning stage, when you are actively collecting data, and after you have finished your analysis. In each of these, try to decide which actions support the integrity of a piece of research, and why you think it does so.</p>
<p>Write down some of your ideas about which actions in the decision tree relate to integrity.</p>
<div class="quarto-figure quarto-figure-left">
<figure class="figure">
<p><img src="images/writing-icon-credit-SmashIcons.png" class="img-fluid quarto-figure quarto-figure-left figure-img" width="50"></p>
</figure>
</div>
<details>
<summary>Show / Hide Discussion</summary>
<p>
<br>
You might have chosen actions like preregistration, positionality statements, or robustness analysis as actions that are related to integrity. It is sometimes tricky to state that an action supports integrity alone, because transparency and integrity often relate to each other: without transparency, we cannot assess whether or not research has integrity.
</p>
</details>
</section>
<section id="quiz" class="level2" data-number="5.6">
<h2 data-number="5.6" class="anchored" data-anchor-id="quiz"><span class="header-section-number">5.6</span> Quiz</h2>
<div class="webex-question">
<div class="webex-check webex-box">
<ol type="1">
<li>What is the purpose of disclosing conflicts of interest in research? (Select one)</li>
</ol>
<div id="webex-2dc038030b53e71b30ce1dbc4720f51c" class="webex-radiogroup" data-answer="aVVPAB8IHANt">
<label><input type="radio" autocomplete="off" name="2dc038030b53e71b30ce1dbc4720f51c"><span>To ensure transparency and prevent bias in the research</span></label><label><input type="radio" autocomplete="off" name="2dc038030b53e71b30ce1dbc4720f51c"><span>To make the research more difficult to understand</span></label><label><input type="radio" autocomplete="off" name="2dc038030b53e71b30ce1dbc4720f51c"><span>To hide potential sources of funding</span></label><label><input type="radio" autocomplete="off" name="2dc038030b53e71b30ce1dbc4720f51c"><span>To increase the complexity of the research findings</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: Disclosing conflicts of interest ensures that you are transparent about any ways in which you may be biased towards the results turning out one way or another.</p>
<ul>
<li>To ensure transparency and prevent bias in the research <b>Correct</b></li>
<li>To make the research more difficult to understand <b>Incorrect</b></li>
<li>To hide potential sources of funding <b>Incorrect</b></li>
<li>To increase the complexity of the research findings <b>Incorrect</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="2" type="1">
<li>Which of the following influences a researcher’s positionality? (Select one or more)</li>
</ol>
<div id="webex-6a820048e3223f3d66d847d9417b2e68" class="webex-radiogroup" data-answer="bVEUAhwAGAlJA28=">
<label><input type="radio" autocomplete="off" name="6a820048e3223f3d66d847d9417b2e68"><span>The presence of flimsy analysis</span></label><label><input type="radio" autocomplete="off" name="6a820048e3223f3d66d847d9417b2e68"><span>The size of the research team</span></label><label><input type="radio" autocomplete="off" name="6a820048e3223f3d66d847d9417b2e68"><span>The geographical location of the research</span></label><label><input type="radio" autocomplete="off" name="6a820048e3223f3d66d847d9417b2e68"><span>The social and political position of the researcher within society</span></label><label><input type="radio" autocomplete="off" name="6a820048e3223f3d66d847d9417b2e68"><span>The political affiliation of the participants</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: Positionality refers to an individual’s social and political position within society, including their identity, background, experiences, and beliefs. This is the best definition. It is also conceivable the place of research could affect a researcher’s positionality. For instance, if a researcher works for a research institute in a country in the Global South, they may have experiences and beliefs relating to that particular country and culture.</p>
<ul>
<li>The presence of flimsy analysis <b>Incorrect</b></li>
<li>The size of the research team <b>Incorrect</b></li>
<li>The geographical location of the research <b>Incorrect</b></li>
<li>The social and political position of the researcher within society <b>Correct</b></li>
<li>The political affiliation of the participants <b>Incorrect</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="3" type="1">
<li>What does robustness refer to in research? (Select one)</li>
</ol>
<div id="webex-6085d986ff1aba605b4051c9c6dc386f" class="webex-radiogroup" data-answer="bQAUBUgIFAY7">
<label><input type="radio" autocomplete="off" name="6085d986ff1aba605b4051c9c6dc386f"><span>The amount of funding received for the research</span></label><label><input type="radio" autocomplete="off" name="6085d986ff1aba605b4051c9c6dc386f"><span>The size of the research team</span></label><label><input type="radio" autocomplete="off" name="6085d986ff1aba605b4051c9c6dc386f"><span>Whether research results hold up under various conditions</span></label><label><input type="radio" autocomplete="off" name="6085d986ff1aba605b4051c9c6dc386f"><span>The complexity of the research findings</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: Robustness refers to the strength and reliability of results. Results can be considered to be more robust, and therefore to have more integrity, if they hold up under various conditions, e.g.: different data analyses.</p>
<ul>
<li>The amount of funding received for the research <b>Incorrect</b></li>
<li>The size of the research team <b>Incorrect</b></li>
<li>Whether research results hold up under various conditions <b>Correct</b></li>
<li>The complexity of the research findings <b>Incorrect</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="4" type="1">
<li>Which of the following is an example of investigating robustness in research? (Select one)</li>
</ol>
<div id="webex-4d570d3371af61a29632d2d6ce04f916" class="webex-radiogroup" data-answer="b1QZBhxUHwNq">
<label><input type="radio" autocomplete="off" name="4d570d3371af61a29632d2d6ce04f916"><span>Sharing data and materials with other researchers</span></label><label><input type="radio" autocomplete="off" name="4d570d3371af61a29632d2d6ce04f916"><span>Running multiple analyses on the data</span></label><label><input type="radio" autocomplete="off" name="4d570d3371af61a29632d2d6ce04f916"><span>Only performing one key analysis on the data</span></label><label><input type="radio" autocomplete="off" name="4d570d3371af61a29632d2d6ce04f916"><span>Writing about potential alternative explanations for the results</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: Running multiple analyses on the data and seeing whether the same conclusions can be drawn is a way to investigate the robustness of a result.</p>
<ul>
<li>Sharing data and materials with other researchers <b>Incorrect</b></li>
<li>Running multiple analyses on the data <b>Correct</b></li>
<li>Only performing one key analysis on the data <b>Incorrect</b></li>
<li>Writing about potential alternative explanations for the results <b>Incorrect</b></li>
</ul>
</div>
</div>
<div class="webex-question">
<div class="webex-check webex-box">
<ol start="5" type="1">
<li>What is multiverse analysis in research? (Select one)</li>
</ol>
<div id="webex-4bd7bc679161c3941f1fc46d2d9154f3" class="webex-radiogroup" data-answer="b1NIB05TGgdk">
<label><input type="radio" autocomplete="off" name="4bd7bc679161c3941f1fc46d2d9154f3"><span>Performing all possible reasonable analyses on the data</span></label><label><input type="radio" autocomplete="off" name="4bd7bc679161c3941f1fc46d2d9154f3"><span>Using new multiverse technology to run analyses on the data</span></label><label><input type="radio" autocomplete="off" name="4bd7bc679161c3941f1fc46d2d9154f3"><span>Exploring multiple universes to find research results</span></label><label><input type="radio" autocomplete="off" name="4bd7bc679161c3941f1fc46d2d9154f3"><span>Running the most complicated analysis possible on the data</span></label>
</div>
</div>
<div class="webex-solution">
<p><strong>Feedback</strong>: Multiverse analysis is what you attempted a mini-version of in Activity 3. As you probably discovered there, multiverse analyses are very hard, and take a long time, but the results can be very interesting and important for understanding how a finding may differ based on different factors.</p>
<ul>
<li>Performing all possible reasonable analyses on the data <b>Correct</b></li>
<li>Using new multiverse technology to run analyses on the data <b>Incorrect</b></li>
<li>Exploring multiple universes to find research results <b>Incorrect</b></li>
<li>Running the most complicated analysis possible on the data <b>Incorrect</b></li>
</ul>
</div>
</div>
</section>
<section id="summary" class="level2" data-number="5.7">
<h2 data-number="5.7" class="anchored" data-anchor-id="summary"><span class="header-section-number">5.7</span> Summary</h2>
<p>This week you explored more aspects of the principle of integrity: conflicts of interest, positionality, and the link between results and conclusions, and robustness.</p>
<p>Positionality is an individual’s social and political position, shaped by factors like identity, background, experiences, values, and beliefs. It can influence the link between results and conclusions: when researchers interpret their data, there is a chance it could colour their interpretation. Potential researcher biases can also impact how researchers write up conclusions following on from their results. It’s important to try to avoid, or at least be explicit about potential biases in your own writing. Finally, robustness is whether or not results hold up in multiple analyses – where they do, we can have more confidence in the results. This helps researchers defend themselves against potential bias.</p>
<p>In Week 6 you’ll be moving on from the principle of integrity to the principle of accessibility.</p>
</section>
</main> <!-- /main -->
<p style="text-align: center; margin: 0; padding: 0;" class="smaller">© 2025 - <a href="www.forrt.org">FORRT</a></p>
<p style="text-align: center;"><span style="color: #808080;" class="smaller"><em>Unless otherwise noted, content on this site is licensed under a <a href="https://creativecommons.org/licenses/by-sa/4.0/">CC BY NC SA 4.0 license</a></em></span></p>
<span style="text-align: center; margin: 0; padding: 0;">
<img src="assets/forrt_frane.png" width="100%">
</span>
<script>
/* definition of content for the buttons */
const webex_buttons = {check_hidden: "<b>✓</b>",
check_hidden_alt: "Check answer",
check_shown: "<b>↰</b>",
check_shown_alt: "Hide check",
check_of_total: "/",
solution: "<b>?</b>",
solution_alt: "Correct solution",
question_next: "<b>↺</b>",
question_next_alt: "Next question",
question_previous: "",
question_previous_alt: ""}
/* update total correct if #webex-total_correct exists */
update_total_correct = function() {
console.log("webex: update total_correct");
document.querySelectorAll(".webex-total_correct").forEach(total => {
p = total.closest(".webex-box");
var correct = p.getElementsByClassName("webex-correct").length;
var solvemes = p.getElementsByClassName("webex-solveme").length;
var radiogroups = p.getElementsByClassName("webex-radiogroup").length;
var selects = p.getElementsByClassName("webex-select").length;
/* no specific class on input node, thus searching via query selector */
var checkboxgroups = p.querySelectorAll("div[class=webex-checkboxgroup] input[type=checkbox]").length
/* show number of correct / total number of answers */
total.innerHTML = correct + " " + webex_buttons.check_of_total + " " + (solvemes + radiogroups + checkboxgroups + selects);
});
}
/* check answers */
check_func = function() {
console.log("webex: check answer");
//var cl = elem.parentElement.classList;
var cl = this.closest(".webex-box").classList;
if (cl.contains("unchecked")) {
cl.remove("unchecked");
this.innerHTML = webex_buttons.check_shown; //"Hide check";
if (webex_buttons.check_shown_alt.length > 0) this.setAttribute("title", webex_buttons.check_shown_alt);
} else {
cl.add("unchecked");
this.innerHTML = webex_buttons.check_hidden; //"Check answer";
if (webex_buttons.check_hidden_alt.length > 0) this.setAttribute("title", webex_buttons.check_hidden_alt);
}
}
/* helper functions for checking the webex ID */
const id_checker = {"fmt": new RegExp("[g-zG-Z]"),
"load": function(e) { return e.getAttribute("id").match("(?<=(-)).*$")[0]; },
"eval": new TextEncoder(),
"dval": new TextDecoder()}
/* Show/hide correct solution */
solution_func = function() {
console.log("webex: show/hide solution");
var div = this.closest(".webex-question").querySelector(".webex-solution");
var cl = div.classList;
if (cl.contains("visible")) {
cl.remove("visible");
} else {
cl.add("visible");
}
}
/* function to check if the real answer is numeric */
convert_to_numeric = function(x) {
if (typeof x == "string") {
/* do nothing */
} else if (x.length == 1) {
/* take first element */
x = x[0]
} else {
return NaN;
}
/* remove spaces for easier parsing */
x = x.replace(/\s/g, "");
/* Define patterns for different formats, note that spaces have been removed above */
const patterns = [
{regex: /^[+-]?\d+(\.\d{3})*,\d+$/, decimal: ",", thousand: "." }, // Format: "1.100.100.100,3"
{regex: /^[+-]?\d+(,\d{3})*\.\d+$/, decimal: ".", thousand: "," }, // Format: "1,100,100,100.3"
{regex: /^[+-]?\d+(\.|\.\d+)?$/, decimal: "." }, // Format: "1100100100.5" or "1100100100."
{regex: /^[+-]?\d+(,|,\d+)$/, decimal: "," } // Format: "1100100100,5" or "1100100100,"
];
/* testing all regular expressions, convert to float if possible */
for (const { regex, decimal, thousand } of patterns) {
if (regex.test(x)) {
let numeric = x;
if (thousand) numeric = numeric.replace(new RegExp(`\\${thousand}`, "g"), "");
numeric = numeric.replace(decimal, ".");
return parseFloat(numeric);
}
}
/* input of length 1 but none of the known formats, return NaN */
return NaN;
}
/* Checking webex ID, returns formalsol */
check_id = function(e) {
/* loading webex ID */
var id = id_checker["load"](e);
/* extracting answer */
let answer = e.dataset.answer;
/* checking unique id, prepare and return array */
if (!id_checker["fmt"].test(id)) {
const a = id_checker["eval"].encode(atob(answer));
const b = id_checker["eval"].encode(id);
const res = a.map((byte, index) => byte^b[index % b.length]);
answer = id_checker["dval"].decode(res);
/* Replacing & with ' before parsing */
answer = answer.replaceAll("'", "'");
}
return JSON.parse(answer);
}
/* function for checking solveme answers */
solveme_func = function(e) {
/* avoid that keyup and keychange twice execute this function within nms = 10
* ms, clearing inputTimer and add timeout */
console.log("webex: check solveme");
/* float, precision for checking numeric answers */
const eps = 0.00000000000001;
/* get last checked user answer */
const question = this.closest(".webex-question")
/* extracting classes */
var cl = this.classList;
var my_answer = this.value;
/* extracting classes */
var cl = this.classList;
/* empty answer? Job done */
if (my_answer == "") {
cl.remove("webex-correct");
cl.remove("webex-incorrect");
return false;
}
/* Find closest (parent) webex-question */
let formalsols = check_id(this)
/* by default we assume the users' answer is incorrect */
var user_answer_correct = false;
/* check if the correct answer is numeric, i.e. if
* formalsols is of length 1 containing one single numeric
* value in a known format, else, NaN is returned */
const num_formalsol = convert_to_numeric(formalsols);
const num_my_answer = convert_to_numeric(my_answer);
/* if the correct answer is numeric (float), the user's answer
* must also be numeric. If not, it is wrong. Else we can
* compare floating point numbers */
if (!isNaN(num_formalsol) && !isNaN(num_my_answer)) {
/* check if the real answer and the user input are numerically the same;
* adding 'delta' to avoid precision issues */
var diff = Math.abs(num_formalsol - num_my_answer);
if (diff < parseFloat(this.dataset.tol) + eps) { user_answer_correct = true; }
/* if the question contains regex, a regular expression is used
* to evaluate the users answer. Allows for multiple regular expressions */
} else if (cl.contains("regex")) {
for (let i = 0; i < formalsols.length; i++) {
let regex = new RegExp(formalsols[i], cl.contains("ignorecase") ? "i" : "");
if (regex.test(my_answer)) { user_answer_correct = true; break; }
}
/* else we evaluate on 'string level', considering the creators preferences
* regarding set options */
} else {
/* modify/prepare answer */
if (cl.contains("ignorecase")) { my_answer = my_answer.toLowerCase(); }
if (cl.contains("nospaces")) { my_answer = my_answer.replace(/ /g, ""); }
/* if the real answer includes user input - correct */
if (formalsols.includes(my_answer)) {
user_answer_correct = true;
// added regex bit
if (cl.contains("regex")) {
answer_regex = RegExp(formalsols.join("|"))
if (answer_regex.test(my_answer)) {
cl.add("webex-correct");
}
}
}
}
if (user_answer_correct) {
cl.add("webex-correct");
cl.remove("webex-incorrect");
} else {
cl.add("webex-incorrect");
cl.remove("webex-correct");
}
update_total_correct();
}
/* function for checking select answers */
select_func = function(e) {
console.log("webex: check select");
var options = [];
Array.from(this.querySelectorAll("option")).forEach((option, index) => {
if (!option.hasAttribute("value") || !option.getAttribute("value") === "blank") {
options.push(option);
}
});
/* get selected option */
function get_selected(options) { return options.findIndex(option => option.selected); }
var option_index = get_selected(options)
/* loading webex ID */
var formalsol = check_id(this)
/* add class for current selection */
this.classList.remove("webex-incorrect");
this.classList.remove("webex-correct");
if (option_index >= 0) {
this.classList.add(formalsol[option_index] === 1 ? "webex-correct" : "webex-incorrect");
}
update_total_correct();
}
/* function for checking radiogroups answers */
radiogroups_func = function(e) {
console.log("webex: check radiogroups");
/* get real answer (binary integer array) */
var id = e.target.getAttribute("name");
var group = document.querySelector(".webex-radiogroup[id='webex-" + id + "']");
var formalsol = check_id(group)
/* check which radiobuttion is selected (user answer) */
function get_checked(radios) {
return radios.findIndex(radios => radios.checked);
}
var radios = Array.from(group.querySelectorAll("input[type='radio']"))
var radio_index = get_checked(radios);
/* remove existing classes */
radios.forEach(radio => {
let cl = radio.closest("label").classList
cl.remove("webex-incorrect");
cl.remove("webex-correct");
});
/* add class for current selection */
let cl = radios[radio_index].closest("label").classList
if (formalsol[radio_index] === 1) {
cl.add("webex-correct");
} else {
cl.add("webex-incorrect");
}
update_total_correct();
}
/* function for checking checkboxgroups answers */
checkboxgroups_func = function(e) {
console.log("webex: check checkboxgroups");
/* get real answer (binary integer array) */
var group = document.querySelector(".webex-checkboxgroup[id='" + this.id + "']");
var formalsol = check_id(group);
/* check which checkbox is checked (user answer) */
function get_checked(group) {
const checkboxes = Array.from(group.querySelectorAll("input[type='checkbox']"));
return checkboxes.reduce((indices, checkbox, index) => {
if (checkbox.checked) indices.push(index);
return indices;
}, []);
}
var checks = Array.from(group.querySelectorAll("input[type='checkbox']"));
var check_index = get_checked(group);
checks.forEach((e, index) => {
var label = e.parentNode;
var ans = formalsol[index];
if ((e.checked && ans === 1) || (!e.checked && ans === 0)) {
label.setAttribute("class", "webex-correct")
} else {
label.setAttribute("class", "webex-incorrect")
}
});
update_total_correct();
}
/* shuffling array (thanks to stack overflow)
* If argument x is an integer we create an integer sequence
* from 0, 1, ..., (x - 1) and return a shuffled version. If
* the input is an array, we simply shuffle it */
shuffle_array = function(x) {
if (Number.isInteger(x) && !isNaN(x)) {
x = Array.from({length: x}, (v, i) => i);
}
let shuffled = x.map(value => ({ value, sort: Math.random() }))
.sort((a, b) => a.sort - b.sort).map(({ value }) => value)
return shuffled;
}
/* ---------------------------------------------------------
* ---------------------------------------------------------
* --------------------------------------------------------- */
window.onload = function() {
//console.log("webex: onload");
/* setting up buttons and actions to show/hide answers */
document.querySelectorAll(".webex-check").forEach(section => {
section.classList.add("unchecked");
/* ul to take up the list items with buttons */
let button_ul = document.createElement("ul");
button_ul.setAttribute("class", "webex-button-list");
section.appendChild(button_ul);
/* button to _check_ if answers given are correct */
let li_check = document.createElement("li");
button_ul.appendChild(li_check); /* add list item to ul */
let btn_check = document.createElement("button");
btn_check.innerHTML = webex_buttons.check_hidden; // "Check answer";
btn_check.setAttribute("class", "webex-button webex-button-check");
if (webex_buttons.check_hidden_alt.length > 0) btn_check.setAttribute("title", webex_buttons.check_hidden_alt);
btn_check.onclick = check_func;
li_check.appendChild(btn_check);
/* span to show current number of points (when _check_ active) */
let spn = document.createElement("span");
spn.classList.add("webex-total_correct");
li_check.appendChild(spn);
/* button to show the _solution_ if there is one */
var has_solution = section.parentNode.querySelectorAll(".webex-solution").length > 0;
if (has_solution) {
let li_solution = document.createElement("li");
button_ul.appendChild(li_solution); /* add list item to ul */
let btn_solution = document.createElement("button");
btn_solution.innerHTML = webex_buttons.solution; // "Correct answer";
btn_solution.setAttribute("class", "webex-button webex-button-solution");
if (webex_buttons.solution_alt.length > 0) btn_solution.setAttribute("title", webex_buttons.solution_alt);
btn_solution.onclick = solution_func;
li_solution.appendChild(btn_solution);
}