Skip to content

Commit cf4ec7c

Browse files
jmafoster1jmafoster1
authored
Merge pull request #296 from CITCOM-project/joss-paper
Joss paper
2 parents 641107f + 8a8ba15 commit cf4ec7c

File tree

3 files changed

+251
-0
lines changed

3 files changed

+251
-0
lines changed

‎.github/workflows/joss.yaml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: JOSS article compilation
2+
on:
3+
push:
4+
paths:
5+
- paper/**
6+
- images/schematic.png
7+
- .github/workflows/draft-pdf.yml
8+
9+
jobs:
10+
paper:
11+
runs-on: ubuntu-latest
12+
name: Paper Draft
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v4
16+
- name: Build draft PDF
17+
uses: openjournals/openjournals-draft-action@master
18+
with:
19+
journal: joss
20+
# This should be the path to the paper within your repo.
21+
paper-path: paper/paper.md
22+
- name: Upload
23+
uses: actions/upload-artifact@v4
24+
with:
25+
name: paper
26+
# This is the output path where Pandoc will write the compiled
27+
# PDF. Note, this should be the same directory as the input
28+
# paper.md
29+
path: paper/paper.pdf

‎paper/paper.bib

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
@article{blobaum2024dowhy,
2+
author = {Patrick Bl{{\"o}}baum and Peter G{{\"o}}tz and Kailash Budhathoki and Atalanti A. Mastakouri and Dominik Janzing},
3+
journal = {Journal of Machine Learning Research},
4+
number = {147},
5+
pages = {1--7},
6+
title = {DoWhy-GCM: An Extension of DoWhy for Causal Inference in Graphical Causal Models},
7+
volume = {25},
8+
year = {2024}
9+
}
10+
11+
@techreport{chen1998metamorphic,
12+
author = {Chen, Tsong Y. and Cheung, Shing C. and Yiu, Shiu Ming},
13+
institution = { The Hong Kong University of Science and Technology},
14+
number = {HKUST-CS98-01},
15+
title = {Metamorphic testing: A new approach for generating next test cases},
16+
year = {1998}
17+
}
18+
19+
@inproceedings{clark2023metamorphic,
20+
author = {Clark, Andrew G. and Foster, Michael and Walkinshaw, Neil and Hierons, Robert M.},
21+
booktitle = {2023 IEEE Conference on Software Testing, Verification and Validation (ICST)},
22+
doi = {10.1109/ICST57152.2023.00023},
23+
keywords = {Software testing;Java;Graphical models;Computer bugs;Software;Test pattern generators;Usability;Metamorphic testing;Causality;DAGs},
24+
number = {},
25+
pages = {153-164},
26+
title = {Metamorphic Testing with Causal Graphs},
27+
volume = {},
28+
year = {2023}
29+
}
30+
31+
@article{clark2023testing,
32+
address = {New York, NY, USA},
33+
articleno = {10},
34+
author = {Clark, Andrew G. and Foster, Michael and Prifling, Benedikt and Walkinshaw, Neil and Hierons, Robert M. and Schmidt, Volker and Turner, Robert D.},
35+
doi = {10.1145/3607184},
36+
issn = {1049-331X},
37+
issue_date = {January 2024},
38+
journal = {ACM Transactions on Software Engineering Methodology},
39+
month = {nov},
40+
number = {1},
41+
numpages = {42},
42+
publisher = {Association for Computing Machinery},
43+
title = {Testing Causality in Scientific Modelling Software},
44+
volume = {33},
45+
year = {2023}
46+
}
47+
48+
@inproceedings{foster2024adequacy,
49+
author = {Foster, Michael and Wild, Christopher and Hierons, Robert M. and Walkinshaw, Neil},
50+
booktitle = {2024 IEEE Conference on Software Testing, Verification and Validation (ICST)},
51+
doi = {10.1109/ICST60714.2024.00023},
52+
number = {},
53+
pages = {161-172},
54+
title = {Causal Test Adequacy},
55+
volume = {},
56+
year = {2024}
57+
}
58+
59+
@inproceedings{guderlei2007smt,
60+
author = {Guderlei, Ralph and Mayer, Johannes},
61+
booktitle = {Seventh International Conference on Quality Software (QSIC 2007)},
62+
doi = {10.1109/QSIC.2007.4385527},
63+
number = {},
64+
pages = {404-409},
65+
title = {Statistical Metamorphic Testing Testing Programs with Random Output by Means of Statistical Hypothesis Tests and Metamorphic Testing},
66+
volume = {},
67+
year = {2007}
68+
}
69+
70+
@book{hernan2020causal,
71+
address = {Boca Raton, FL},
72+
author = {Hern{\'a}n, Miguel A and Robins, James M},
73+
publisher = {Chapman \& Hall/CRC},
74+
title = {Causal {I}nference: {What} if},
75+
year = {2020}
76+
}
77+
78+
@book{pearl2009causality,
79+
address = {Cambridge},
80+
author = {Judea Pearl},
81+
day = {14},
82+
isbn = {9780521895606},
83+
month = {09},
84+
pagecount = {464},
85+
publisher = {Cambridge university press},
86+
subtitle = {Models, Reasoning, and Infernce},
87+
title = {Causality},
88+
year = {2009}
89+
}
90+
91+
@misc{sharma2020dowhy,
92+
archiveprefix = {arXiv},
93+
author = {Amit Sharma and Emre Kiciman},
94+
eprint = {2011.04216},
95+
primaryclass = {stat.ME},
96+
title = {DoWhy: An End-to-End Library for Causal Inference},
97+
url = {https://arxiv.org/abs/2011.04216},
98+
year = {2020}
99+
}
100+
101+
@article{somers2024configuration,
102+
author = {Somers, Richard and Walkinshaw, Neil and Hierons, Robert and Elliott, Jackie and Iqbal, Ahmed and Walkinshaw, Emma},
103+
doi = {10.2139/ssrn.4732706},
104+
publisher = {Elsevier BV},
105+
title = {Configuration Testing of an Artificial Pancreas System Using a Digital Twin},
106+
year = {2024}
107+
}
108+
109+
@article{textor2017dagitty,
110+
author = {Textor, Johannes and van der Zander, Benito and Gilthorpe, Mark S. and Liśkiewicz, Maciej and Ellison, George T.H.},
111+
doi = {10.1093/ije/dyw341},
112+
issn = {1464-3685},
113+
journal = {International Journal of Epidemiology},
114+
month = {jan},
115+
pages = {dyw341},
116+
publisher = {Oxford University Press (OUP)},
117+
title = {Robust causal inference using directed acyclic graphs: the R package ‘dagitty’},
118+
year = {2017}
119+
}

‎paper/paper.md

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
---
2+
title: 'The Causal Testing Framework'
3+
tags:
4+
- Python
5+
- causal testing
6+
- causal inference
7+
- causality
8+
- software testing
9+
- metamorphic testing
10+
authors:
11+
- name: Michael Foster
12+
orcid: 0000-0001-8233-9873
13+
affiliation: 1
14+
corresponding: true
15+
- name: Andrew Clark
16+
orcid: 0000-0002-6830-0566
17+
affiliation: 1
18+
- name: Christopher Wild
19+
orcid: 0009-0009-1195-1497
20+
affiliation: 1
21+
- name: Farhad Allian
22+
orcid: 0000-0002-4569-0370
23+
affiliation: 1
24+
- name: Robert Turner
25+
orcid: 0000-0002-1353-1404
26+
affiliation: 1
27+
- name: Richard Somers
28+
orcid: 0009-0009-1195-1497
29+
affiliation: 1
30+
- name: Nicholas Latimer
31+
orcid: 0000-0001-5304-5585
32+
affiliation: 1
33+
- name: Neil Walkinshaw
34+
orcid: 0000-0003-2134-6548
35+
affiliation: 1
36+
- name: Rob Hierons
37+
orcid: 0000-0003-2134-6548
38+
affiliation: 1
39+
affiliations:
40+
- name: University of Sheffield, UK
41+
index: 1
42+
date: 2 December 2024
43+
bibliography: paper.bib
44+
---
45+
46+
# Summary
47+
Scientific models possess several properties that make them notoriously difficult to test, including a complex input space, long execution times, and non-determinism, rendering existing testing techniques impractical.
48+
In fields such as epidemiology, where researchers seek answers to challenging causal questions, a statistical methodology known as Causal Inference (CI) [@pearl2009causality,@hernan2020causal] has addressed similar problems, enabling the inference of causal conclusions from noisy, biased, and sparse observational data instead of costly randomised trials.
49+
CI works by using domain knowledge to identify and mitigate for biases in the data, enabling them to answer causal questions that concern the effect of changing some feature on the observed outcome.
50+
The Causal Testing Framework (CTF) is a software testing framework that uses CI techniques to establish causal effects between software variables from pre-existing runtime data rather than having to collect bespoke, highly curated datasets especially for testing.
51+
52+
# Statement of need
53+
Metamorphic Testing [@chen1998metamorphic] is a popular technique for testing computational models (and other traditionally "hard to test" software).
54+
Test goals are expressed as _metamorphic relations_ that specify how changing an input in a particular way should affect the software output.
55+
Nondeterministic software can be tested using Statistical Metamorphic Testing [@guderlei2007smt], which uses statistical tests over multiple executions of the software to determine whether the specified metamorphic relations hold.
56+
However, this requires the software to be executed repeatedly for each set of parameters of interest, so is computationally expensive, and is constrained to testing properties over software inputs that can be directly and precisely controlled.
57+
Statistical Metamorphic Testing cannot be used to test properties that relate internal variables or outputs to each other, since these cannot be controlled a priori.
58+
59+
By employing domain knowledge in the form of a causal graph --- a lightweight model specifying the expected relationships between key software variables --- the CTF overcomes the limitations of Statistical Metamorphic Testing by enabling models to be tested using pre-existing runtime data.
60+
The CTF is written in Python but is language agnostic in terms of the system under test.
61+
All that is required is a set of properties to be validated, a causal model, and a set of software runtime data.
62+
63+
# Causal Testing
64+
Causal Testing [@clark2023testing] has four main steps, outlined in Figure \ref{fig:schematic}.
65+
Firstly, the user supplies a causal model, which takes the form of a directed acyclic graph (DAG) [@pearl2009causality] where an edge $X \to Y$ represents variable $X$ having a direct causal effect on variable $Y$.
66+
Secondly, the user supplies a set of causal properties to be tested.
67+
Such properties can be generated from the causal DAG [@clark2023metamorphic]: for each $X \to Y$ edge, a test to validate the presence of a causal effect is generated, and for each missing edge, a test to validate independence is generated.
68+
The user may also refine tests to validate the nature of a particular relationship.
69+
Next, the user supplies a set of runtime data in the form of a table with each column representing a variable and rows containing the value of each variable for a particular run of the software.
70+
Finally, the CTF automatically validates the causal properties by using the causal DAG to identify a statistical estimand [@pearl2009causality] (essentially a set of features in the data which must be controlled for), calculate a causal effect estimate from the supplied data, and validating this against the expected causal relationship.
71+
72+
![Causal Testing workflow.\label{fig:schematic}](../images/schematic.png)
73+
74+
## Test Adequacy
75+
Because the properties being tested are completely separate from the data used to validate them, traditional coverage-based metrics are not appropriate here.
76+
The CTF instead evaluates the adequacy of a particular dataset by calculating a statistical metric [@foster2024adequacy] based on the stability of the causal effect estimate, with numbers closer to zero representing more adequate data.
77+
78+
## Missing Variables
79+
Causal Testing works by using the causal DAG to identify the variables that need to be statistically controlled for to remove their biassing effect on the causal estimate.
80+
This typically means we need to know their values.
81+
However, where such biassing variables are not recorded in the data, the Causal Testing Framework can still sometimes estimate unbiased causal effects by using Instrumental Variables [@hernan2020causal], an advanced Causal Inference technique.
82+
83+
## Feedback Over Time
84+
Many scientific models involve iterating several interacting processes over time.
85+
These processes often feed into each other, and can create feedback cycles.
86+
Traditional CI cannot handle this, however the CTF uses a family of advanced CI techniques, called g-methods [@hernan2020causal], to enable the estimation of causal effects even when there are feedback cycles between variables.
87+
88+
# Related Work
89+
The Dagitty tool [@textor2017dagitty] is a browser-based environment for creating, editing, and analysing causal graphs.
90+
There is also an R package for local use, but Dagitty cannot be used to estimate causal effects.
91+
For this, doWhy [@sharma2020dowhy; @blobaum2024dowhy] is a free, open source Python package, and [cStruture](https://cstructure.dev) is a paid low code CI platform.
92+
However, these packages are intended for general CI.
93+
Neither explicitly supports causal software testing, nor do they support temporal feedback loops.
94+
95+
# Ongoing and Future Research
96+
The CTF is the subject of several publications [@clark2023metamorphic; @clark2023testing; @foster2024adequacy; @somers2024configuration].
97+
We are also in the process of preparing scientific publications concerning how the CTF handles missing variables and feedback over time.
98+
Furthermore, we are working to develop a plug-in for the [DAFNI platform](https://www.dafni.ac.uk/) to enable national-scale infrastructure models to be easily tested.
99+
100+
# Acknowledgements
101+
This work was supported by the EPSRC CITCoM grant EP/T030526/1.
102+
103+
# References

0 commit comments

Comments
 (0)