Skip to content

Commit e6a6674

Browse files
james-cockaynejames-cockayne
authored
Merge pull request #306 from answerdigital/bugfix/out_of_domain_concept_map
Fix bug where a concept could be mapped to an unknown concept rather …
2 parents 3edb4da + 4d1faf4 commit e6a6674

23 files changed

+327
-79
lines changed

‎OmopTransformer/ConceptDeviceSelector.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using OmopTransformer.Annotations;
2+
using OmopTransformer.ConceptResolution;
23
using OmopTransformer.Transformation;
34

45
namespace OmopTransformer;

‎OmopTransformer/ConceptResolution/ConceptCodeMapRow.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
namespace OmopTransformer.ConceptResolution;
2+
3+
public class ConceptCodeMapRow
4+
{
5+
public int source_concept_id { get; init; }
6+
public int? target_concept_id { get; init; }
7+
8+
public string? source_domain_id { get; init; }
9+
public string? target_domain_id { get; init; }
10+
public byte mapped_from_standard { get; init; }
11+
}

‎OmopTransformer/ConceptResolution/ConceptRelationshipRow.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace OmopTransformer.ConceptResolution;
2+
3+
internal class ConceptRelationshipRow
4+
{
5+
public int concept_id { get; init; }
6+
public int device_concept_id { get; init; }
7+
}

‎OmopTransformer/ConceptResolution/IStandardConceptResolverDataProvider.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace OmopTransformer.ConceptResolution;
2+
3+
public interface IStandardConceptResolverDataProvider
4+
{
5+
Dictionary<int, IGrouping<int, ConceptCodeMapRow>> GetMappings();
6+
Dictionary<int, IReadOnlyCollection<int>> GetDevices();
7+
}

OmopTransformer/StandardConceptResolver.cs renamed to OmopTransformer/ConceptResolution/StandardConceptResolver.cs

Lines changed: 17 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,82 +1,34 @@
11
using System.Collections.Concurrent;
2-
using Dapper;
3-
using DuckDB.NET.Data;
42
using Microsoft.Extensions.Logging;
5-
using Microsoft.Extensions.Options;
63

7-
namespace OmopTransformer;
4+
namespace OmopTransformer.ConceptResolution;
85

96
internal class StandardConceptResolver
107
{
11-
private readonly Configuration _configuration;
128
private readonly ILogger<StandardConceptResolver> _logger;
9+
private readonly IStandardConceptResolverDataProvider _dataProvider;
1310

14-
private Dictionary<int, IGrouping<int, Row>>? _mappings;
11+
private Dictionary<int, IGrouping<int, ConceptCodeMapRow>>? _mappings;
1512
private Dictionary<int, IReadOnlyCollection<int>>? _devicesByConceptId;
1613

1714
private readonly object _loadingLock = new();
1815
private DomainMapResults? _domainMappingResults;
1916

20-
public StandardConceptResolver(IOptions<Configuration> configuration, ILogger<StandardConceptResolver> logger)
17+
public StandardConceptResolver(ILogger<StandardConceptResolver> logger, IStandardConceptResolverDataProvider dataProvider)
2118
{
2219
_logger = logger;
23-
_configuration = configuration.Value;
24-
}
25-
26-
private Dictionary<int, IGrouping<int, Row>> GetMappings()
27-
{
28-
_logger.LogInformation("Loading mappings codes.");
29-
30-
var connection = new DuckDBConnection(_configuration.ConnectionString!);
31-
connection.Open();
32-
33-
var results = connection.Query<Row>("select * from omop_staging.concept_code_map", CancellationToken.None);
34-
35-
return
36-
results
37-
.GroupBy(row => row.source_concept_id!)
38-
.ToDictionary(row => row.Key!);
39-
}
40-
41-
private Dictionary<int, IReadOnlyCollection<int>> GetDevices()
42-
{
43-
_logger.LogInformation("Loading concept device relationships.");
44-
45-
var connection = new DuckDBConnection(_configuration.ConnectionString!);
46-
connection.Open();
47-
48-
var results =
49-
connection
50-
.Query<ConceptRelationshipRow>(
51-
@"select distinct
52-
cm.source_concept_id as concept_id,
53-
device.concept_id as device_concept_id
54-
from omop_staging.concept_code_map cm
55-
inner join cdm.concept_relationship cr
56-
on cm.target_concept_id = cr.concept_id_1
57-
inner join cdm.concept device
58-
on cr.concept_id_2 = device.concept_id
59-
where device.standard_concept = 'S'
60-
and cr.relationship_id like '%device%'
61-
and device.domain_id = 'Device'",
62-
CancellationToken.None);
63-
64-
return
65-
results
66-
.GroupBy(group => group.concept_id)
67-
.ToDictionary(
68-
row => row.Key,
69-
row => (IReadOnlyCollection<int>)row.Select(map => map.device_concept_id).ToList());
20+
_dataProvider = dataProvider;
7021
}
7122

23+
7224
private void EnsureMapping()
7325
{
7426
if (_mappings != null)
7527
return;
7628

7729
lock (_loadingLock)
7830
{
79-
_mappings ??= GetMappings();
31+
_mappings ??= _dataProvider.GetMappings();
8032

8133
if (_mappings.Count == 0)
8234
{
@@ -89,37 +41,37 @@ private void EnsureDeviceMapping()
8941
{
9042
lock (_loadingLock)
9143
{
92-
_devicesByConceptId ??= GetDevices();
44+
_devicesByConceptId ??= _dataProvider.GetDevices();
9345
}
9446
}
9547

96-
private static int? ResolveConcept(Row row, string domain)
48+
private static int? ResolveConcept(ConceptCodeMapRow conceptCodeMapRow, string domain)
9749
{
9850
const int unknownConceptId = 0;
9951

10052
// Standard or non standard with relationship with standard concept in concept correct domain
101-
if (row.target_domain_id == domain && row.target_concept_id.HasValue)
53+
if (conceptCodeMapRow.target_domain_id == domain && conceptCodeMapRow.target_concept_id.HasValue)
10254
{
103-
return row.target_concept_id;
55+
return conceptCodeMapRow.target_concept_id;
10456
}
10557

10658

10759
//Non standard or Standard concept in wrong domain
10860

109-
if (row.target_domain_id != domain && row.target_concept_id.HasValue)
61+
if (conceptCodeMapRow.target_domain_id != domain && conceptCodeMapRow.target_concept_id.HasValue)
11062
{
11163
return null;
11264
}
11365

11466
//Non standard concept in wrong domain with no relationship to standard
11567

116-
if (row.mapped_from_standard == 0 && row.source_domain_id != domain && row.target_concept_id.HasValue == false)
68+
if (conceptCodeMapRow.mapped_from_standard == 0 && conceptCodeMapRow.source_domain_id != domain && conceptCodeMapRow.target_concept_id.HasValue == false)
11769
{
11870
return null;
11971
}
12072

12173
// Non standard concept in correct domain with no relationship to standard
122-
if (row.mapped_from_standard == 0 && row.source_domain_id == domain && row.target_concept_id.HasValue == false)
74+
if (conceptCodeMapRow.mapped_from_standard == 0 && conceptCodeMapRow.source_domain_id == domain && conceptCodeMapRow.target_concept_id.HasValue == false)
12375
{
12476
return unknownConceptId;
12577
}
@@ -197,6 +149,8 @@ public int[] GetConcepts(int conceptId, string? domain)
197149
if (row.target_domain_id != null)
198150
_domainMappingResults.Record(row.target_domain_id!);
199151
}
152+
153+
return resolvedConcepts;
200154
}
201155

202156
return unknownConcept;
@@ -221,22 +175,6 @@ public void PrintLogsAndResetLogger()
221175
_domainMappingResults = null;
222176
}
223177

224-
private class Row
225-
{
226-
public int source_concept_id { get; init; }
227-
public int? target_concept_id { get; init; }
228-
229-
public string? source_domain_id { get; init; }
230-
public string? target_domain_id { get; init; }
231-
public byte mapped_from_standard { get; init; }
232-
}
233-
234-
private class ConceptRelationshipRow
235-
{
236-
public int concept_id { get; init; }
237-
public int device_concept_id { get; init; }
238-
}
239-
240178
private class DomainMapResults
241179
{
242180
public DomainMapResults(string intendedDomain)
@@ -266,7 +204,7 @@ public void PrintResults(ILogger<StandardConceptResolver> logger)
266204

267205
foreach (var domainCount in _domainCounts.OrderByDescending(count => count.Value))
268206
{
269-
var percentage = (domainCount.Value * 100d) / total;
207+
var percentage = domainCount.Value * 100d / total;
270208
logText += $" - Domain: {domainCount.Key}. Count: {domainCount.Value}. Percentage: {Math.Round(percentage, 2)}%." + Environment.NewLine;
271209
}
272210

‎OmopTransformer/ConceptResolution/StandardConceptResolverDataProvider.cs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
using Dapper;
2+
using DuckDB.NET.Data;
3+
using Microsoft.Extensions.Logging;
4+
using Microsoft.Extensions.Options;
5+
6+
namespace OmopTransformer.ConceptResolution;
7+
8+
internal class StandardConceptResolverDataProvider : IStandardConceptResolverDataProvider
9+
{
10+
private readonly Configuration _configuration;
11+
private readonly ILogger<StandardConceptResolver> _logger;
12+
13+
public StandardConceptResolverDataProvider(IOptions<Configuration> configuration, ILogger<StandardConceptResolver> logger)
14+
{
15+
_logger = logger;
16+
_configuration = configuration.Value;
17+
}
18+
19+
public Dictionary<int, IGrouping<int, ConceptCodeMapRow>> GetMappings()
20+
{
21+
_logger.LogInformation("Loading mappings codes.");
22+
23+
var connection = new DuckDBConnection(_configuration.ConnectionString!);
24+
connection.Open();
25+
26+
var results = connection.Query<ConceptCodeMapRow>("select * from omop_staging.concept_code_map", CancellationToken.None);
27+
28+
return
29+
results
30+
.GroupBy(row => row.source_concept_id!)
31+
.ToDictionary(row => row.Key!);
32+
}
33+
34+
public Dictionary<int, IReadOnlyCollection<int>> GetDevices()
35+
{
36+
_logger.LogInformation("Loading concept device relationships.");
37+
38+
var connection = new DuckDBConnection(_configuration.ConnectionString!);
39+
connection.Open();
40+
41+
var results =
42+
connection
43+
.Query<ConceptRelationshipRow>(
44+
@"select distinct
45+
cm.source_concept_id as concept_id,
46+
device.concept_id as device_concept_id
47+
from omop_staging.concept_code_map cm
48+
inner join cdm.concept_relationship cr
49+
on cm.target_concept_id = cr.concept_id_1
50+
inner join cdm.concept device
51+
on cr.concept_id_2 = device.concept_id
52+
where device.standard_concept = 'S'
53+
and cr.relationship_id like '%device%'
54+
and device.domain_id = 'Device'",
55+
CancellationToken.None);
56+
57+
return
58+
results
59+
.GroupBy(group => group.concept_id)
60+
.ToDictionary(
61+
row => row.Key,
62+
row => (IReadOnlyCollection<int>)row.Select(map => map.device_concept_id).ToList());
63+
}
64+
}

‎OmopTransformer/OxfordGP/OxfordGPTransformer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using OmopTransformer.OxfordGP.DrugExposure;
2020
using OmopTransformer.OxfordGP.DeviceExposure;
2121
using OmopTransformer.Transformation;
22+
using OmopTransformer.ConceptResolution;
2223

2324
namespace OmopTransformer.OxfordGP;
2425

‎OmopTransformer/OxfordLab/OxfordLabTransformer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Microsoft.Extensions.Logging;
2+
using OmopTransformer.ConceptResolution;
23
using OmopTransformer.Omop;
34
using OmopTransformer.Omop.Measurement;
45
using OmopTransformer.Omop.Observation;

‎OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using OmopTransformer.Transformation;
55
using OmopTransformer.Omop.DrugExposure;
66
using OmopTransformer.OxfordPrescribing.DrugExposureWithSnomed;
7+
using OmopTransformer.ConceptResolution;
78

89
namespace OmopTransformer.OxfordPrescribing;
910

‎OmopTransformer/OxfordSpineDeath/OxfordSpineDeathTransformer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Microsoft.Extensions.Logging;
2+
using OmopTransformer.ConceptResolution;
23
using OmopTransformer.Omop;
34
using OmopTransformer.Omop.Death;
45
using OmopTransformer.OxfordSpineDeath.Death;

0 commit comments

Comments
 (0)