Skip to content

Commit de26a6c

Browse files
jeromekelleherjeromekelleher
committed
Catch table overflows.
Closes #157
1 parent 9035bda commit de26a6c

File tree

4 files changed

+187
-2
lines changed

4 files changed

+187
-2
lines changed

‎c/tests/test_tables.c‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2103,6 +2103,99 @@ test_load_reindex(void)
21032103
tsk_treeseq_free(&ts);
21042104
}
21052105

2106+
static void
2107+
test_table_overflow(void)
2108+
{
2109+
int ret;
2110+
tsk_table_collection_t tables;
2111+
tsk_size_t max_rows = ((tsk_size_t) INT32_MAX) + 1;
2112+
2113+
ret = tsk_table_collection_init(&tables, 0);
2114+
CU_ASSERT_EQUAL_FATAL(ret, 0);
2115+
2116+
/* Simulate overflows */
2117+
tables.individuals.max_rows = max_rows;
2118+
tables.individuals.num_rows = max_rows;
2119+
ret = tsk_individual_table_add_row(&tables.individuals, 0, 0, 0, NULL, 0);
2120+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2121+
2122+
tables.nodes.max_rows = max_rows;
2123+
tables.nodes.num_rows = max_rows;
2124+
ret = tsk_node_table_add_row(&tables.nodes, 0, 0, 0, 0, NULL, 0);
2125+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2126+
2127+
tables.edges.max_rows = max_rows;
2128+
tables.edges.num_rows = max_rows;
2129+
ret = tsk_edge_table_add_row(&tables.edges, 0, 0, 0, 0);
2130+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2131+
2132+
tables.migrations.max_rows = max_rows;
2133+
tables.migrations.num_rows = max_rows;
2134+
ret = tsk_migration_table_add_row(&tables.migrations, 0, 0, 0, 0, 0, 0);
2135+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2136+
2137+
tables.sites.max_rows = max_rows;
2138+
tables.sites.num_rows = max_rows;
2139+
ret = tsk_site_table_add_row(&tables.sites, 0, 0, 0, 0, 0);
2140+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2141+
2142+
tables.mutations.max_rows = max_rows;
2143+
tables.mutations.num_rows = max_rows;
2144+
ret = tsk_mutation_table_add_row(&tables.mutations, 0, 0, 0, 0, 0, 0, 0);
2145+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2146+
2147+
tables.provenances.max_rows = max_rows;
2148+
tables.provenances.num_rows = max_rows;
2149+
ret = tsk_provenance_table_add_row(&tables.provenances, 0, 0, 0, 0);
2150+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2151+
2152+
tables.populations.max_rows = max_rows;
2153+
tables.populations.num_rows = max_rows;
2154+
ret = tsk_population_table_add_row(&tables.populations, 0, 0);
2155+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_TABLE_OVERFLOW);
2156+
2157+
tsk_table_collection_free(&tables);
2158+
}
2159+
2160+
static void
2161+
test_column_overflow(void)
2162+
{
2163+
int ret;
2164+
tsk_table_collection_t tables;
2165+
tsk_size_t too_big = ((tsk_size_t) INT32_MAX) + 2;
2166+
2167+
ret = tsk_table_collection_init(&tables, 0);
2168+
CU_ASSERT_EQUAL_FATAL(ret, 0);
2169+
2170+
ret = tsk_individual_table_add_row(&tables.individuals, 0, NULL, too_big, NULL, 0);
2171+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2172+
ret = tsk_individual_table_add_row(&tables.individuals, 0, NULL, 0, NULL, too_big);
2173+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2174+
2175+
ret = tsk_node_table_add_row(&tables.nodes, 0, 0, 0, 0, NULL, too_big);
2176+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2177+
2178+
ret = tsk_site_table_add_row(&tables.sites, 0, NULL, too_big, NULL, 0);
2179+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2180+
ret = tsk_site_table_add_row(&tables.sites, 0, NULL, 0, NULL, too_big);
2181+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2182+
2183+
ret = tsk_mutation_table_add_row(&tables.mutations, 0, 0, 0, NULL, 0, NULL, too_big);
2184+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2185+
ret = tsk_mutation_table_add_row(&tables.mutations, 0, 0, 0, NULL, too_big, NULL, 0);
2186+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2187+
2188+
ret = tsk_provenance_table_add_row(&tables.provenances, NULL, too_big, NULL, 0);
2189+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2190+
ret = tsk_provenance_table_add_row(&tables.provenances, NULL, 0, NULL, too_big);
2191+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2192+
2193+
ret = tsk_population_table_add_row(&tables.populations, NULL, too_big);
2194+
CU_ASSERT_EQUAL_FATAL(ret, TSK_ERR_COLUMN_OVERFLOW);
2195+
2196+
tsk_table_collection_free(&tables);
2197+
}
2198+
21062199
int
21072200
main(int argc, char **argv)
21082201
{
@@ -2129,8 +2222,11 @@ main(int argc, char **argv)
21292222
{"test_dump_load_empty", test_dump_load_empty},
21302223
{"test_dump_load_unsorted", test_dump_load_unsorted},
21312224
{"test_load_reindex", test_load_reindex},
2225+
{"test_table_overflow", test_table_overflow},
2226+
{"test_column_overflow", test_column_overflow},
21322227
{NULL, NULL},
21332228
};
21342229

2230+
21352231
return test_main(tests, argc, argv);
21362232
}

‎c/tskit/core.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,12 @@ tsk_strerror_internal(int err)
293293
case TSK_ERR_TABLES_NOT_INDEXED:
294294
ret = "Table collection must be indexed";
295295
break;
296+
case TSK_ERR_TABLE_OVERFLOW:
297+
ret = "Table too large; cannot allocate more than 2**31 rows.";
298+
break;
299+
case TSK_ERR_COLUMN_OVERFLOW:
300+
ret = "Table column too large; cannot be more than 2**31 bytes.";
301+
break;
296302

297303
/* Limitations */
298304
case TSK_ERR_ONLY_INFINITE_SITES:

‎c/tskit/core.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ of tskit.
189189
#define TSK_ERR_BAD_TABLE_POSITION -700
190190
#define TSK_ERR_BAD_SEQUENCE_LENGTH -701
191191
#define TSK_ERR_TABLES_NOT_INDEXED -702
192+
#define TSK_ERR_TABLE_OVERFLOW -703
193+
#define TSK_ERR_COLUMN_OVERFLOW -704
192194

193195
/* Limitations */
194196
#define TSK_ERR_ONLY_INFINITE_SITES -800

‎c/tskit/tables.c‎

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,15 @@ typedef struct {
5656
int type;
5757
} write_table_col_t;
5858

59+
/* Returns true if adding the specifiec number of rows would result in overflow.
60+
* Tables can support indexes from 0 to INT32_MAX, and therefore have at most
61+
* INT32_MAX + 1 rows */
62+
static bool
63+
check_overflow(tsk_size_t current_size, tsk_size_t additional_rows)
64+
{
65+
size_t new_size = (size_t) current_size + (size_t) additional_rows;
66+
return new_size > ((size_t) INT32_MAX) + 1;
67+
}
5968

6069
static int
6170
read_table_cols(kastore_t *store, read_table_col_t *read_cols, size_t num_cols)
@@ -167,6 +176,10 @@ tsk_individual_table_expand_main_columns(tsk_individual_table_t *self,
167176
tsk_size_t increment = TSK_MAX(additional_rows, self->max_rows_increment);
168177
tsk_size_t new_size = self->max_rows + increment;
169178

179+
if (check_overflow(self->max_rows, increment)) {
180+
ret = TSK_ERR_TABLE_OVERFLOW;
181+
goto out;
182+
}
170183
if ((self->num_rows + additional_rows) > self->max_rows) {
171184
ret = expand_column((void **) &self->flags, new_size, sizeof(tsk_flags_t));
172185
if (ret != 0) {
@@ -192,10 +205,13 @@ static int
192205
tsk_individual_table_expand_location(tsk_individual_table_t *self, tsk_size_t additional_length)
193206
{
194207
int ret = 0;
195-
tsk_size_t increment = TSK_MAX(additional_length,
196-
self->max_location_length_increment);
208+
tsk_size_t increment = TSK_MAX(additional_length, self->max_location_length_increment);
197209
tsk_size_t new_size = self->max_location_length + increment;
198210

211+
if (check_overflow(self->location_length, increment)) {
212+
ret = TSK_ERR_COLUMN_OVERFLOW;
213+
goto out;
214+
}
199215
if ((self->location_length + additional_length) > self->max_location_length) {
200216
ret = expand_column((void **) &self->location, new_size, sizeof(double));
201217
if (ret != 0) {
@@ -215,6 +231,10 @@ tsk_individual_table_expand_metadata(tsk_individual_table_t *self, tsk_size_t ad
215231
self->max_metadata_length_increment);
216232
tsk_size_t new_size = self->max_metadata_length + increment;
217233

234+
if (check_overflow(self->metadata_length, increment)) {
235+
ret = TSK_ERR_COLUMN_OVERFLOW;
236+
goto out;
237+
}
218238
if ((self->metadata_length + additional_length) > self->max_metadata_length) {
219239
ret = expand_column((void **) &self->metadata, new_size, sizeof(char));
220240
if (ret != 0) {
@@ -646,6 +666,10 @@ tsk_node_table_expand_main_columns(tsk_node_table_t *self, tsk_size_t additional
646666
tsk_size_t increment = TSK_MAX(additional_rows, self->max_rows_increment);
647667
tsk_size_t new_size = self->max_rows + increment;
648668

669+
if (check_overflow(self->max_rows, increment)) {
670+
ret = TSK_ERR_TABLE_OVERFLOW;
671+
goto out;
672+
}
649673
if ((self->num_rows + additional_rows) > self->max_rows) {
650674
ret = expand_column((void **) &self->flags, new_size, sizeof(tsk_flags_t));
651675
if (ret != 0) {
@@ -682,6 +706,10 @@ tsk_node_table_expand_metadata(tsk_node_table_t *self, tsk_size_t additional_len
682706
self->max_metadata_length_increment);
683707
tsk_size_t new_size = self->max_metadata_length + increment;
684708

709+
if (check_overflow(self->metadata_length, increment)) {
710+
ret = TSK_ERR_COLUMN_OVERFLOW;
711+
goto out;
712+
}
685713
if ((self->metadata_length + additional_length) > self->max_metadata_length) {
686714
ret = expand_column((void **) &self->metadata, new_size, sizeof(char));
687715
if (ret != 0) {
@@ -1056,6 +1084,10 @@ tsk_edge_table_expand_columns(tsk_edge_table_t *self, size_t additional_rows)
10561084
(tsk_size_t) additional_rows, self->max_rows_increment);
10571085
tsk_size_t new_size = self->max_rows + increment;
10581086

1087+
if (check_overflow(self->max_rows, increment)) {
1088+
ret = TSK_ERR_TABLE_OVERFLOW;
1089+
goto out;
1090+
}
10591091
if ((self->num_rows + additional_rows) > self->max_rows) {
10601092
ret = expand_column((void **) &self->left, new_size, sizeof(double));
10611093
if (ret != 0) {
@@ -1321,6 +1353,10 @@ tsk_site_table_expand_main_columns(tsk_site_table_t *self, tsk_size_t additional
13211353
tsk_size_t increment = TSK_MAX(additional_rows, self->max_rows_increment);
13221354
tsk_size_t new_size = self->max_rows + increment;
13231355

1356+
if (check_overflow(self->max_rows, increment)) {
1357+
ret = TSK_ERR_TABLE_OVERFLOW;
1358+
goto out;
1359+
}
13241360
if ((self->num_rows + additional_rows) > self->max_rows) {
13251361
ret = expand_column((void **) &self->position, new_size, sizeof(double));
13261362
if (ret != 0) {
@@ -1350,6 +1386,10 @@ tsk_site_table_expand_ancestral_state(tsk_site_table_t *self, size_t additional_
13501386
self->max_ancestral_state_length_increment);
13511387
tsk_size_t new_size = self->max_ancestral_state_length + increment;
13521388

1389+
if (check_overflow(self->ancestral_state_length, increment)) {
1390+
ret = TSK_ERR_COLUMN_OVERFLOW;
1391+
goto out;
1392+
}
13531393
if ((self->ancestral_state_length + additional_length)
13541394
> self->max_ancestral_state_length) {
13551395
ret = expand_column((void **) &self->ancestral_state, new_size, sizeof(char));
@@ -1370,6 +1410,10 @@ tsk_site_table_expand_metadata(tsk_site_table_t *self, size_t additional_length)
13701410
self->max_metadata_length_increment);
13711411
tsk_size_t new_size = self->max_metadata_length + increment;
13721412

1413+
if (check_overflow(self->metadata_length, increment)) {
1414+
ret = TSK_ERR_COLUMN_OVERFLOW;
1415+
goto out;
1416+
}
13731417
if ((self->metadata_length + additional_length)
13741418
> self->max_metadata_length) {
13751419
ret = expand_column((void **) &self->metadata, new_size, sizeof(char));
@@ -1776,6 +1820,11 @@ tsk_mutation_table_expand_main_columns(tsk_mutation_table_t *self, size_t additi
17761820
tsk_size_t increment = (tsk_size_t) TSK_MAX(additional_rows, self->max_rows_increment);
17771821
tsk_size_t new_size = self->max_rows + increment;
17781822

1823+
1824+
if (check_overflow(self->max_rows, increment)) {
1825+
ret = TSK_ERR_TABLE_OVERFLOW;
1826+
goto out;
1827+
}
17791828
if ((self->num_rows + additional_rows) > self->max_rows) {
17801829
ret = expand_column((void **) &self->site, new_size, sizeof(tsk_id_t));
17811830
if (ret != 0) {
@@ -1813,6 +1862,10 @@ tsk_mutation_table_expand_derived_state(tsk_mutation_table_t *self, size_t addit
18131862
self->max_derived_state_length_increment);
18141863
tsk_size_t new_size = self->max_derived_state_length + increment;
18151864

1865+
if (check_overflow(self->derived_state_length, increment)) {
1866+
ret = TSK_ERR_COLUMN_OVERFLOW;
1867+
goto out;
1868+
}
18161869
if ((self->derived_state_length + additional_length)
18171870
> self->max_derived_state_length) {
18181871
ret = expand_column((void **) &self->derived_state, new_size, sizeof(char));
@@ -1833,6 +1886,10 @@ tsk_mutation_table_expand_metadata(tsk_mutation_table_t *self, size_t additional
18331886
self->max_metadata_length_increment);
18341887
tsk_size_t new_size = self->max_metadata_length + increment;
18351888

1889+
if (check_overflow(self->metadata_length, increment)) {
1890+
ret = TSK_ERR_COLUMN_OVERFLOW;
1891+
goto out;
1892+
}
18361893
if ((self->metadata_length + additional_length)
18371894
> self->max_metadata_length) {
18381895
ret = expand_column((void **) &self->metadata, new_size, sizeof(char));
@@ -2262,6 +2319,10 @@ tsk_migration_table_expand(tsk_migration_table_t *self, size_t additional_rows)
22622319
(tsk_size_t) additional_rows, self->max_rows_increment);
22632320
tsk_size_t new_size = self->max_rows + increment;
22642321

2322+
if (check_overflow(self->max_rows, increment)) {
2323+
ret = TSK_ERR_TABLE_OVERFLOW;
2324+
goto out;
2325+
}
22652326
if ((self->num_rows + additional_rows) > self->max_rows) {
22662327
ret = expand_column((void **) &self->left, new_size, sizeof(double));
22672328
if (ret != 0) {
@@ -2558,6 +2619,10 @@ tsk_population_table_expand_main_columns(tsk_population_table_t *self, tsk_size_
25582619
tsk_size_t increment = TSK_MAX(additional_rows, self->max_rows_increment);
25592620
tsk_size_t new_size = self->max_rows + increment;
25602621

2622+
if (check_overflow(self->max_rows, increment)) {
2623+
ret = TSK_ERR_TABLE_OVERFLOW;
2624+
goto out;
2625+
}
25612626
if ((self->num_rows + additional_rows) > self->max_rows) {
25622627
ret = expand_column((void **) &self->metadata_offset, new_size + 1,
25632628
sizeof(tsk_size_t));
@@ -2578,6 +2643,10 @@ tsk_population_table_expand_metadata(tsk_population_table_t *self, tsk_size_t ad
25782643
self->max_metadata_length_increment);
25792644
tsk_size_t new_size = self->max_metadata_length + increment;
25802645

2646+
if (check_overflow(self->metadata_length, increment)) {
2647+
ret = TSK_ERR_COLUMN_OVERFLOW;
2648+
goto out;
2649+
}
25812650
if ((self->metadata_length + additional_length) > self->max_metadata_length) {
25822651
ret = expand_column((void **) &self->metadata, new_size, sizeof(char));
25832652
if (ret != 0) {
@@ -2890,6 +2959,10 @@ tsk_provenance_table_expand_main_columns(tsk_provenance_table_t *self, tsk_size_
28902959
tsk_size_t increment = TSK_MAX(additional_rows, self->max_rows_increment);
28912960
tsk_size_t new_size = self->max_rows + increment;
28922961

2962+
if (check_overflow(self->max_rows, increment)) {
2963+
ret = TSK_ERR_TABLE_OVERFLOW;
2964+
goto out;
2965+
}
28932966
if ((self->num_rows + additional_rows) > self->max_rows) {
28942967
ret = expand_column((void **) &self->timestamp_offset, new_size + 1,
28952968
sizeof(tsk_size_t));
@@ -2915,6 +2988,10 @@ tsk_provenance_table_expand_timestamp(tsk_provenance_table_t *self, tsk_size_t a
29152988
self->max_timestamp_length_increment);
29162989
tsk_size_t new_size = self->max_timestamp_length + increment;
29172990

2991+
if (check_overflow(self->timestamp_length, increment)) {
2992+
ret = TSK_ERR_COLUMN_OVERFLOW;
2993+
goto out;
2994+
}
29182995
if ((self->timestamp_length + additional_length) > self->max_timestamp_length) {
29192996
ret = expand_column((void **) &self->timestamp, new_size, sizeof(char));
29202997
if (ret != 0) {
@@ -2934,6 +3011,10 @@ tsk_provenance_table_expand_provenance(tsk_provenance_table_t *self, tsk_size_t
29343011
self->max_record_length_increment);
29353012
tsk_size_t new_size = self->max_record_length + increment;
29363013

3014+
if (check_overflow(self->record_length, increment)) {
3015+
ret = TSK_ERR_COLUMN_OVERFLOW;
3016+
goto out;
3017+
}
29373018
if ((self->record_length + additional_length) > self->max_record_length) {
29383019
ret = expand_column((void **) &self->record, new_size, sizeof(char));
29393020
if (ret != 0) {

0 commit comments

Comments
 (0)