diff --git a/OmopTransformer/COSD/Colorectal/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml b/OmopTransformer/COSD/Colorectal/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml index 328cb158..58a8de98 100644 --- a/OmopTransformer/COSD/Colorectal/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml +++ b/OmopTransformer/COSD/Colorectal/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml @@ -22,8 +22,7 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - + Patient NHS Number diff --git a/OmopTransformer/COSD/Core/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml b/OmopTransformer/COSD/Core/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml index 374c4c01..c7c3913d 100644 --- a/OmopTransformer/COSD/Core/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml +++ b/OmopTransformer/COSD/Core/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml @@ -1,6 +1,5 @@  - -- fail with cosddates as ( select Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber, diff --git a/OmopTransformer/ConceptDeviceSelector.cs b/OmopTransformer/ConceptDeviceSelector.cs index 67f0c1e8..8d805c33 100644 --- a/OmopTransformer/ConceptDeviceSelector.cs +++ b/OmopTransformer/ConceptDeviceSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Finds related devices for a concept.")] -internal class ConceptDeviceSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class ConceptDeviceSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/ConceptLookup.cs b/OmopTransformer/ConceptLookup.cs index 294ff30d..5e5de8a3 100644 --- a/OmopTransformer/ConceptLookup.cs +++ b/OmopTransformer/ConceptLookup.cs @@ -41,16 +41,21 @@ private Dictionary GetCodes() public virtual string FormatCode(string code) => code; - public int? GetConceptCode(string? code) + public int GetConceptCode(string? code) { - lock (_loadingLock) + const int unknownConceptId = 0; + + if (_mappings == null) { - _mappings ??= GetCodes(); + lock (_loadingLock) + { + _mappings ??= GetCodes(); + } } if (code == null) { - return null; + return unknownConceptId; } var formatCode = FormatCode(code); @@ -70,7 +75,7 @@ private Dictionary GetCodes() } } - return null; + return unknownConceptId; } } diff --git a/OmopTransformer/ConceptResolver.cs b/OmopTransformer/ConceptResolver.cs deleted file mode 100644 index 977bca12..00000000 --- a/OmopTransformer/ConceptResolver.cs +++ /dev/null @@ -1,176 +0,0 @@ -using Dapper; -using DuckDB.NET.Data; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace OmopTransformer; - -internal class ConceptResolver -{ - private readonly Configuration _configuration; - private readonly ILogger _logger; - - private Dictionary>? _mappings; - private Dictionary>? _devicesByConceptId; - - private readonly object _loadingLock = new(); - - private readonly Dictionary _unableToMapByFrequency = new(); - private readonly object _unableToMapByFrequencyLock = new(); - - public ConceptResolver(IOptions configuration, ILogger logger) - { - _logger = logger; - _configuration = configuration.Value; - } - - private Dictionary> GetMappings() - { - _logger.LogInformation("Loading mappings codes."); - - var connection = new DuckDBConnection(_configuration.ConnectionString!); - connection.Open(); - - var results = connection.Query("select * from omop_staging.concept_code_map", CancellationToken.None); - - return - results - .GroupBy(row => row.source_concept_id!) - .ToDictionary(row => row.Key!); - } - - private Dictionary> GetDevices() - { - _logger.LogInformation("Loading concept device relationships."); - - var connection = new DuckDBConnection(_configuration.ConnectionString!); - connection.Open(); - - var results = - connection - .Query( - @"select distinct - cm.source_concept_id as concept_id, - device.concept_id as device_concept_id - from omop_staging.concept_code_map cm - inner join cdm.concept_relationship cr - on cm.target_concept_id = cr.concept_id_1 - inner join cdm.concept device - on cr.concept_id_2 = device.concept_id - where device.standard_concept = 'S' - and cr.relationship_id like '%device%' - and device.domain_id = 'Device'", - CancellationToken.None); - - return - results - .GroupBy(group => group.concept_id) - .ToDictionary( - row => row.Key, - row => (IReadOnlyCollection)row.Select(map => map.device_concept_id).ToList()); - } - - private void EnsureMapping() - { - if (_mappings != null) - return; - - lock (_loadingLock) - { - _mappings ??= GetMappings(); - - if (_mappings.Count == 0) - { - throw new InvalidOperationException("concept_code_map table is empty. Call stored procedure omop_staging.generate_concept_code_map first."); - } - } - } - - private void EnsureDeviceMapping() - { - lock (_loadingLock) - { - _devicesByConceptId ??= GetDevices(); - } - } - - public int[] GetConcepts(int conceptId, string? domain) - { - EnsureMapping(); - - if (_mappings!.TryGetValue(conceptId, out var value)) - { - return - value - .Where(row => domain == null || row.domain_id!.Equals(domain, StringComparison.OrdinalIgnoreCase)) - .Select(row => row.target_concept_id) - .ToArray(); - } - - lock (_unableToMapByFrequencyLock) - { - if (_unableToMapByFrequency.TryGetValue(conceptId, out int count)) - { - _unableToMapByFrequency[conceptId] = ++count; - } - else - { - _unableToMapByFrequency.Add(conceptId, 1); - } - } - - return new int[] { }; - } - - public IReadOnlyCollection GetConceptDevices(int conceptId) - { - EnsureDeviceMapping(); - - if (_devicesByConceptId!.TryGetValue(conceptId, out var devices)) - { - return devices.ToArray(); - } - - return new int[] { }; - } - - public void ResetErrors() - { - lock (_unableToMapByFrequencyLock) - { - _unableToMapByFrequency.Clear(); - } - } - - public void PrintErrors() - { - lock (_unableToMapByFrequencyLock) - { - if (_unableToMapByFrequency.Any()) - { - string log = "Top 10 unmappable concept codes. Unmappble concepts cannot be recorded." + Environment.NewLine; - - foreach (var error in _unableToMapByFrequency.OrderByDescending(map => map.Value).Take(10)) - { - log += $"- concept id: {error.Key}. error count: {error.Value}." + Environment.NewLine; - } - - _logger.LogWarning(log); - } - } - } - - private class Row - { - public int source_concept_id { get; init; } - public int target_concept_id { get; init; } - public string? domain_id { get; init; } - public byte mapped_from_standard { get; init; } - } - - private class ConceptRelationshipRow - { - public int concept_id { get; init; } - public int device_concept_id { get; init; } - } -} \ No newline at end of file diff --git a/OmopTransformer/Init/Initiation.cs b/OmopTransformer/Init/Initiation.cs index 9c7a4be2..efdaac27 100644 --- a/OmopTransformer/Init/Initiation.cs +++ b/OmopTransformer/Init/Initiation.cs @@ -635,16 +635,13 @@ invalid_count integer NOT NULL CREATE TABLE omop_staging.concept_code_map( source_concept_code varchar(50) NOT NULL, source_concept_id integer NOT NULL, - target_concept_id integer NOT NULL, - domain_id varchar(50) NOT NULL, - mapped_from_standard bit NOT NULL, - constraint PK_omop_staging_concept_code_map_source_concept_id_target_concept_id PRIMARY KEY - ( - source_concept_id, - target_concept_id - ) + target_concept_id integer NULL, + source_domain_id varchar(50) NOT NULL, + target_domain_id varchar(50) NULL, + mapped_from_standard tinyint NOT NULL ); + CREATE TABLE omop_staging.cosd_staging_81( SubmissionName varchar(200) NOT NULL, FileName varchar(200) NOT NULL, @@ -2530,7 +2527,8 @@ insert into omop_staging.concept_code_map source_concept_code, source_concept_id, target_concept_id, - domain_id, + source_domain_id, + target_domain_id, mapped_from_standard ) with InvalidConceptMap as ( @@ -2538,7 +2536,8 @@ with InvalidConceptMap as ( c1.concept_code as source_concept_code, c1.concept_id as source_concept_id, c2.concept_id as target_concept_id, - c2.domain_id + c1.domain_id as source_domain_id, + c2.domain_id as target_domain_id from cdm.concept c1 inner join cdm.concept_relationship cr on c1.concept_id = cr.concept_id_1 @@ -2560,7 +2559,8 @@ and current_date between c2.valid_start_date and c2.valid_end_date c.concept_code as source_concept_code, c.concept_id as source_concept_id, c.concept_id as target_concept_id, - c.domain_id as domain_id, + c.domain_id as target_domain_id, + c.domain_id as source_domain_id, 1 as mapped_from_standard from cdm.concept c where c.standard_concept is not null @@ -2573,7 +2573,8 @@ and current_date between c.valid_start_date and c.valid_end_date icm.source_concept_code, icm.source_concept_id, icm.target_concept_id as concept_id, - c.domain_id, + icm.source_domain_id, + icm.target_domain_id, 0 as mapped_from_standard from InvalidConceptMap icm inner join cdm.concept c @@ -2586,7 +2587,8 @@ where icm.target_concept_id is not null source_concept_code, source_concept_id, target_concept_id, - domain_id, + source_domain_id, + target_domain_id, mapped_from_standard from Mapped; @@ -2595,14 +2597,16 @@ insert into omop_staging.concept_code_map source_concept_code, source_concept_id, target_concept_id, - domain_id, + source_domain_id, + target_domain_id, mapped_from_standard ) select c.concept_code as source_concept_code, c.concept_id as source_concept_id, rxnormConcept.concept_id as target_concept_id, - 'Drug' as domain_id, + 'Drug' as source_domain_id, + 'Drug' as target_domain_id, 1 as mapped_from_standard from cdm.concept c inner join cdm.concept dmdConcept @@ -2625,6 +2629,30 @@ from omop_staging.concept_code_map ccm and rxnormConcept.standard_concept = 'S' and rxnormConcept.domain_id = 'Drug' and rxnormConcept.vocabulary_id in ('RxNorm', 'RxNorm Extension'); + +insert into omop_staging.concept_code_map +( + source_concept_code, + source_concept_id, + target_concept_id, + source_domain_id, + target_domain_id, + mapped_from_standard +) +select + c.concept_code, + c.concept_id, + null, + c.domain_id, + null, + 0 +from cdm.concept c +where c.standard_concept is null + and not exists ( + select * + from omop_staging.concept_code_map ccm + where ccm.source_concept_id = c.concept_id + ) "); diff --git a/OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs b/OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs index c55575c5..7c239cd2 100644 --- a/OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs +++ b/OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs @@ -123,6 +123,7 @@ where r.RecordConnectionIdentifier is not null and co.person_id = p.person_id and co.RecordConnectionIdentifier = r.RecordConnectionIdentifier and co.condition_concept_id = r.condition_concept_id + and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value) ) and not exists ( select 1 @@ -131,6 +132,7 @@ where r.RecordConnectionIdentifier is null and co.condition_concept_id = r.condition_concept_id and co.condition_start_date = r.condition_start_date and co.person_id = p.person_id + and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value) ); diff --git a/OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs b/OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs index bf5fcdcf..577901d5 100644 --- a/OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs +++ b/OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs @@ -165,6 +165,7 @@ from cdm.device_exposure vo and vo.device_concept_id = r.device_concept_id and vo.device_exposure_start_date = r.device_exposure_start_date and vo.device_exposure_end_date = r.device_exposure_end_date + and (vo.device_concept_id != 0 or ((vo.device_source_value is null and r.device_source_value is null) or vo.device_source_value = r.device_source_value)) ) ); diff --git a/OmopTransformer/Omop/DrugExposure/DrugExposureRecorder.cs b/OmopTransformer/Omop/DrugExposure/DrugExposureRecorder.cs index baa73b3c..9ff5351d 100644 --- a/OmopTransformer/Omop/DrugExposure/DrugExposureRecorder.cs +++ b/OmopTransformer/Omop/DrugExposure/DrugExposureRecorder.cs @@ -157,6 +157,7 @@ from cdm.drug_exposure vo where vo.RecordConnectionIdentifier = r.RecordConnectionIdentifier and vo.person_id = p.person_id and vo.drug_concept_id = r.drug_concept_id + ) ) or @@ -168,6 +169,7 @@ from cdm.drug_exposure vo where vo.person_id = p.person_id and vo.drug_concept_id = r.drug_concept_id and vo.drug_exposure_start_date = r.drug_exposure_start_date + and (vo.drug_concept_id != 0 or ((vo.drug_source_value is null and r.drug_source_value is null) or vo.drug_source_value = r.drug_source_value)) ) ); diff --git a/OmopTransformer/Omop/Measurement/MeasurementRecorder.cs b/OmopTransformer/Omop/Measurement/MeasurementRecorder.cs index 54ee5302..839de62e 100644 --- a/OmopTransformer/Omop/Measurement/MeasurementRecorder.cs +++ b/OmopTransformer/Omop/Measurement/MeasurementRecorder.cs @@ -144,6 +144,7 @@ from cdm.measurement m and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id) and r.RecordConnectionIdentifier is null and r.HospitalProviderSpellNumber is null + and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value)) ) and not exists ( select 1 @@ -154,6 +155,7 @@ from cdm.measurement m and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id) and r.RecordConnectionIdentifier is not null and m.RecordConnectionIdentifier = r.RecordConnectionIdentifier + and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value)) ) and not exists ( select 1 @@ -164,6 +166,7 @@ from cdm.measurement m and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id) and r.HospitalProviderSpellNumber is not null and m.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber + and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value)) ); truncate table omop_staging.measurement_row;", diff --git a/OmopTransformer/Omop/Observation/ObservationRecorder.cs b/OmopTransformer/Omop/Observation/ObservationRecorder.cs index dd59df01..48723a7f 100644 --- a/OmopTransformer/Omop/Observation/ObservationRecorder.cs +++ b/OmopTransformer/Omop/Observation/ObservationRecorder.cs @@ -158,7 +158,8 @@ r.RecordConnectionIdentifier is not null and o.observation_concept_id = r.observation_concept_id and o.RecordConnectionIdentifier = r.RecordConnectionIdentifier and (r.HospitalProviderSpellNumber is null or o.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber) and - (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id) + (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id) and + (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string)) ) or ( @@ -166,7 +167,8 @@ r.RecordConnectionIdentifier is null and o.person_id = p.person_id and o.observation_date = r.observation_date and o.observation_concept_id = r.observation_concept_id and - o.observation_source_concept_id = r.observation_source_concept_id + o.observation_source_concept_id = r.observation_source_concept_id and + (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string)) ) ); diff --git a/OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs b/OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs index b66789fe..f94cab1d 100644 --- a/OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs +++ b/OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs @@ -129,14 +129,17 @@ r.RecordConnectionIdentifier is not null and vo.RecordConnectionIdentifier = r.RecordConnectionIdentifier and vo.person_id = p.person_id and vo.procedure_date = r.procedure_date and - vo.procedure_concept_id = r.procedure_concept_id + vo.procedure_concept_id = r.procedure_concept_id and + (vo.procedure_concept_id != 0 or ((vo.procedure_source_value is null and r.procedure_source_value is null) or vo.procedure_source_value = r.procedure_source_value)) + ) or ( r.RecordConnectionIdentifier is null and vo.procedure_date = r.procedure_date and vo.procedure_concept_id = r.procedure_concept_id and - vo.person_id = p.person_id + vo.person_id = p.person_id and + (vo.procedure_concept_id != 0 or ((vo.procedure_source_value is null and r.procedure_source_value is null) or vo.procedure_source_value = r.procedure_source_value)) ) ); diff --git a/OmopTransformer/OxfordGP/OxfordGPTransformer.cs b/OmopTransformer/OxfordGP/OxfordGPTransformer.cs index 47b87892..a8bfac67 100644 --- a/OmopTransformer/OxfordGP/OxfordGPTransformer.cs +++ b/OmopTransformer/OxfordGP/OxfordGPTransformer.cs @@ -27,7 +27,7 @@ internal class OxfordGPTransformer : Transformer private readonly ILocationRecorder _locationRecorder; private readonly IPersonRecorder _personRecorder; private readonly IDeathRecorder _deathRecorder; - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder; private readonly IConditionOccurrenceRecorder _conditionOccurrenceRecorder; private readonly IVisitOccurrenceRecorder _visitOccurrenceRecorder; @@ -42,7 +42,7 @@ public OxfordGPTransformer( ILocationRecorder locationRecorder, IPersonRecorder personRecorder, IDeathRecorder deathRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IRunAnalysisRecorder runAnalysisRecorder, ILoggerFactory loggerFactory, IProcedureOccurrenceRecorder procedureOccurrenceRecorder, @@ -126,7 +126,5 @@ await Transform( "Oxford GP Device Exposure", runId, cancellationToken); - - _conceptResolver.PrintErrors(); } } diff --git a/OmopTransformer/OxfordLab/OxfordLabTransformer.cs b/OmopTransformer/OxfordLab/OxfordLabTransformer.cs index 4bf1149a..fe4fb1f4 100644 --- a/OmopTransformer/OxfordLab/OxfordLabTransformer.cs +++ b/OmopTransformer/OxfordLab/OxfordLabTransformer.cs @@ -10,7 +10,7 @@ namespace OmopTransformer.OxfordLab; internal class OxfordLabTransformer : Transformer { - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly IMeasurementRecorder _measurementRecorder; private readonly IObservationRecorder _observationRecorder; @@ -20,7 +20,7 @@ public OxfordLabTransformer( IRecordProvider recordProvider, IMeasurementRecorder measurementRecorder, IObservationRecorder observationRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IRunAnalysisRecorder runAnalysisRecorder, ILoggerFactory loggerFactory) : base(recordTransformer, transformOptions, @@ -49,7 +49,5 @@ await Transform( "Oxford Lab General Comments", runId, cancellationToken); - - _conceptResolver.PrintErrors(); } } diff --git a/OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs b/OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs index 6e607bf9..f28a93c9 100644 --- a/OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs +++ b/OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs @@ -9,7 +9,7 @@ namespace OmopTransformer.OxfordPrescribing; internal class OxfordPrescribingTransformer : Transformer { - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly IDrugExposureRecorder _drugExposureRecorder; public OxfordPrescribingTransformer( @@ -17,7 +17,7 @@ public OxfordPrescribingTransformer( TransformOptions transformOptions, IRecordProvider recordProvider, IDrugExposureRecorder drugExposureRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IRunAnalysisRecorder runAnalysisRecorder, ILoggerFactory loggerFactory) : base(recordTransformer, transformOptions, @@ -45,7 +45,5 @@ await Transform(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); IHostEnvironment env = builder.Environment; diff --git a/OmopTransformer/SUS/AE/SusAETransformer.cs b/OmopTransformer/SUS/AE/SusAETransformer.cs index 993b416a..fbbebb62 100644 --- a/OmopTransformer/SUS/AE/SusAETransformer.cs +++ b/OmopTransformer/SUS/AE/SusAETransformer.cs @@ -37,7 +37,7 @@ internal class SusAETransformer : Transformer private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder; private readonly IDeviceExposureRecorder _deviceExposureRecorder; private readonly IObservationRecorder _observationRecorder; - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly ICareSiteRecorder _careSiteRecorder; public SusAETransformer( @@ -52,7 +52,7 @@ public SusAETransformer( IVisitDetailRecorder visitDetailRecorder, IDeathRecorder deathRecorder, IProcedureOccurrenceRecorder procedureOccurrenceRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IDeviceExposureRecorder deviceExposureRecorder, IObservationRecorder observationRecorder, IRunAnalysisRecorder runAnalysisRecorder, @@ -157,7 +157,5 @@ await Transform( "SUS AE CareSite", runId, cancellationToken); - - _conceptResolver.PrintErrors(); } } diff --git a/OmopTransformer/SUS/APC/SusAPCTransformer.cs b/OmopTransformer/SUS/APC/SusAPCTransformer.cs index 76341a6f..5348cd8a 100644 --- a/OmopTransformer/SUS/APC/SusAPCTransformer.cs +++ b/OmopTransformer/SUS/APC/SusAPCTransformer.cs @@ -54,7 +54,7 @@ internal class SusAPCTransformer : Transformer private readonly IDeathRecorder _deathRecorder; private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder; private readonly IObservationRecorder _observationRecorder; - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly ICareSiteRecorder _careSiteRecorder; private readonly IProviderRecorder _providerRecorder; private readonly IDeviceExposureRecorder _deviceExposureRecorder; @@ -73,7 +73,7 @@ public SusAPCTransformer( IVisitDetailRecorder visitDetailRecorder, IDeathRecorder deathRecorder, IProcedureOccurrenceRecorder procedureOccurrenceRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IDeviceExposureRecorder deviceExposureRecorder, IObservationRecorder observationRecorder, IRunAnalysisRecorder runAnalysisRecorder, @@ -272,7 +272,5 @@ await Transform( "SUS APC sus_OP_OPCSProcedure Observations", runId, cancellationToken); - - _conceptResolver.PrintErrors(); } } diff --git a/OmopTransformer/SUS/OP/SusOPTransformer.cs b/OmopTransformer/SUS/OP/SusOPTransformer.cs index b7d694a0..9b7d887b 100644 --- a/OmopTransformer/SUS/OP/SusOPTransformer.cs +++ b/OmopTransformer/SUS/OP/SusOPTransformer.cs @@ -42,7 +42,7 @@ internal class SusOPTransformer : Transformer private readonly IDeathRecorder _deathRecorder; private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder; private readonly IObservationRecorder _observationRecorder; - private readonly ConceptResolver _conceptResolver; + private readonly StandardConceptResolver _conceptResolver; private readonly ICareSiteRecorder _careSiteRecorder; private readonly IProviderRecorder _providerRecorder; private readonly IDeviceExposureRecorder _deviceExposureRecorder; @@ -61,7 +61,7 @@ public SusOPTransformer( IVisitDetailRecorder visitDetailRecorder, IDeathRecorder deathRecorder, IProcedureOccurrenceRecorder procedureOccurrenceRecorder, - ConceptResolver conceptResolver, + StandardConceptResolver conceptResolver, IObservationRecorder observationRecorder, IDeviceExposureRecorder deviceExposureRecorder, IRunAnalysisRecorder runAnalysisRecorder, @@ -186,7 +186,5 @@ await Transform( "SUS OP sus_OP_OPCSProcedure Observations", runId, cancellationToken); - - _conceptResolver.PrintErrors(); } } \ No newline at end of file diff --git a/OmopTransformer/StandardConceptResolver.cs b/OmopTransformer/StandardConceptResolver.cs new file mode 100644 index 00000000..ca90fa6e --- /dev/null +++ b/OmopTransformer/StandardConceptResolver.cs @@ -0,0 +1,276 @@ +using System.Collections.Concurrent; +using Dapper; +using DuckDB.NET.Data; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace OmopTransformer; + +internal class StandardConceptResolver +{ + private readonly Configuration _configuration; + private readonly ILogger _logger; + + private Dictionary>? _mappings; + private Dictionary>? _devicesByConceptId; + + private readonly object _loadingLock = new(); + private DomainMapResults? _domainMappingResults; + + public StandardConceptResolver(IOptions configuration, ILogger logger) + { + _logger = logger; + _configuration = configuration.Value; + } + + private Dictionary> GetMappings() + { + _logger.LogInformation("Loading mappings codes."); + + var connection = new DuckDBConnection(_configuration.ConnectionString!); + connection.Open(); + + var results = connection.Query("select * from omop_staging.concept_code_map", CancellationToken.None); + + return + results + .GroupBy(row => row.source_concept_id!) + .ToDictionary(row => row.Key!); + } + + private Dictionary> GetDevices() + { + _logger.LogInformation("Loading concept device relationships."); + + var connection = new DuckDBConnection(_configuration.ConnectionString!); + connection.Open(); + + var results = + connection + .Query( + @"select distinct + cm.source_concept_id as concept_id, + device.concept_id as device_concept_id + from omop_staging.concept_code_map cm + inner join cdm.concept_relationship cr + on cm.target_concept_id = cr.concept_id_1 + inner join cdm.concept device + on cr.concept_id_2 = device.concept_id + where device.standard_concept = 'S' + and cr.relationship_id like '%device%' + and device.domain_id = 'Device'", + CancellationToken.None); + + return + results + .GroupBy(group => group.concept_id) + .ToDictionary( + row => row.Key, + row => (IReadOnlyCollection)row.Select(map => map.device_concept_id).ToList()); + } + + private void EnsureMapping() + { + if (_mappings != null) + return; + + lock (_loadingLock) + { + _mappings ??= GetMappings(); + + if (_mappings.Count == 0) + { + throw new InvalidOperationException("concept_code_map table is empty. Call stored procedure omop_staging.generate_concept_code_map first."); + } + } + } + + private void EnsureDeviceMapping() + { + lock (_loadingLock) + { + _devicesByConceptId ??= GetDevices(); + } + } + + private static int? ResolveConcept(Row row, string domain) + { + const int unknownConceptId = 0; + + // Standard or non standard with relationship with standard concept in concept correct domain + if (row.target_domain_id == domain && row.target_concept_id.HasValue) + { + return row.target_concept_id; + } + + + //Non standard or Standard concept in wrong domain + + if (row.target_domain_id != domain && row.target_concept_id.HasValue) + { + return null; + } + + //Non standard concept in wrong domain with no relationship to standard + + if (row.mapped_from_standard == 0 && row.source_domain_id != domain && row.target_concept_id.HasValue == false) + { + return null; + } + + // Non standard concept in correct domain with no relationship to standard + if (row.mapped_from_standard == 0 && row.source_domain_id == domain && row.target_concept_id.HasValue == false) + { + return unknownConceptId; + } + + return null; + } + + public int[] GetConcepts(int conceptId, string? domain) + { + EnsureMapping(); + + const int unknownConceptId = 0; + + var unknownConcept = new int[] { unknownConceptId }; + + //| Domain Constraint | Input | Source Concept Id | Concept Id | + //| -------------------| -------| ------------------| ------------| + //| No | Unknown concept | 0 | 0 | + //| No | Standard concept | 456 | 456 | + //| No | Non standard concept with no relationship to standard | 123 | 0 | + //| No | Non standard concept with relationship to standard | 123 | 456 | + //| Yes | Unknown concept | 0 | 0 | + //| Yes | Standard concept correct domain | 456 | 456 | + //| Yes | Non standard concept with relationship to standard in correct domain | 123 | 456 | + //| Yes | Non standard concept in correct domain with no relationship to standard | 123 | 0 | + //| Yes | Standard concept wrong domain | null | null | + //| Yes | Non standard concept in wrong domain with no relationship to standard | null | null | + //| Yes | Non standard concept with relationship to standard in wrong domain | null | null + + + if (domain == null) + { + if (_mappings!.TryGetValue(conceptId, out var value)) + { + if (value.Any(row => row.target_concept_id.HasValue == false)) + { + // Non standard concept with no relationship to standard + + return unknownConcept; + } + + // Standard concept or Non standard concept with relationship to standard + return + value + .Select(row => row.target_concept_id!.Value) + .ToArray(); + } + + // Unknown concept + return unknownConcept; + } + else + { + if (_mappings!.TryGetValue(conceptId, out var value)) + { + + var resolvedConcepts = + value + .Select(row => ResolveConcept(row, domain)) + .Where(result => result.HasValue) + .Select(result => result!.Value) + .ToArray(); + + if (resolvedConcepts.Length > 0) + { + return resolvedConcepts; + } + + + _domainMappingResults ??= new DomainMapResults(domain); + + // Log domain of standard version of input concept, if exists + foreach (var row in value) + { + if (row.target_domain_id != null) + _domainMappingResults.Record(row.target_domain_id!); + } + } + + return unknownConcept; + } + } + + public IReadOnlyCollection GetConceptDevices(int conceptId) + { + EnsureDeviceMapping(); + + if (_devicesByConceptId!.TryGetValue(conceptId, out var devices)) + { + return devices.ToArray(); + } + + return new int[] { }; + } + + public void PrintLogsAndResetLogger() + { + _domainMappingResults?.PrintResults(_logger); + _domainMappingResults = null; + } + + private class Row + { + public int source_concept_id { get; init; } + public int? target_concept_id { get; init; } + + public string? source_domain_id { get; init; } + public string? target_domain_id { get; init; } + public byte mapped_from_standard { get; init; } + } + + private class ConceptRelationshipRow + { + public int concept_id { get; init; } + public int device_concept_id { get; init; } + } + + private class DomainMapResults + { + public DomainMapResults(string intendedDomain) + { + _intendedDomain = intendedDomain; + } + + private readonly ConcurrentDictionary _domainCounts = new(); + private readonly string _intendedDomain; + + public void Record(string domain) + { + _domainCounts.AddOrUpdate(domain, 1, (_, count) => count + 1); + } + + public void PrintResults(ILogger logger) + { + if (_domainCounts.Count == 0) + return; + + if (_domainCounts.Count == 1 && _domainCounts.ContainsKey(_intendedDomain)) + return; + + long total = _domainCounts.Sum(count => count.Value); + + string logText = $"Source concepts intended for domain {_intendedDomain} were mapped to the following domains:" + Environment.NewLine; + + foreach (var domainCount in _domainCounts.OrderByDescending(count => count.Value)) + { + var percentage = (domainCount.Value * 100d) / total; + logText += $" - Domain: {domainCount.Key}. Count: {domainCount.Value}. Percentage: {Math.Round(percentage, 2)}%." + Environment.NewLine; + } + + logger.LogInformation(logText); + } + } +} \ No newline at end of file diff --git a/OmopTransformer/StandardConceptSelector.cs b/OmopTransformer/StandardConceptSelector.cs index e019d430..32e5ef0a 100644 --- a/OmopTransformer/StandardConceptSelector.cs +++ b/OmopTransformer/StandardConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Converts concepts to standard concepts.")] -internal class StandardConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardConditionConceptSelector.cs b/OmopTransformer/StandardConditionConceptSelector.cs index 288d2f31..92078afc 100644 --- a/OmopTransformer/StandardConditionConceptSelector.cs +++ b/OmopTransformer/StandardConditionConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `condition` domain.")] -internal class StandardConditionConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardConditionConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardDeviceConceptSelector.cs b/OmopTransformer/StandardDeviceConceptSelector.cs index 861eeef9..0b3cd8e7 100644 --- a/OmopTransformer/StandardDeviceConceptSelector.cs +++ b/OmopTransformer/StandardDeviceConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `device` domain.")] -internal class StandardDeviceConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardDeviceConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardDrugConceptSelector.cs b/OmopTransformer/StandardDrugConceptSelector.cs index 35d7b16a..fb319373 100644 --- a/OmopTransformer/StandardDrugConceptSelector.cs +++ b/OmopTransformer/StandardDrugConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `drug` domain.")] -internal class StandardDrugConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardDrugConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardMeasurementConceptSelector.cs b/OmopTransformer/StandardMeasurementConceptSelector.cs index 89c22a19..a94a5f14 100644 --- a/OmopTransformer/StandardMeasurementConceptSelector.cs +++ b/OmopTransformer/StandardMeasurementConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `measurement` domain.")] -internal class StandardMeasurementConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardMeasurementConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardObservationConceptSelector.cs b/OmopTransformer/StandardObservationConceptSelector.cs index fde983c0..528b1051 100644 --- a/OmopTransformer/StandardObservationConceptSelector.cs +++ b/OmopTransformer/StandardObservationConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `Observation` domain.")] -internal class StandardObservationConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardObservationConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/StandardProcedureConceptSelector.cs b/OmopTransformer/StandardProcedureConceptSelector.cs index fdd31956..20eabea6 100644 --- a/OmopTransformer/StandardProcedureConceptSelector.cs +++ b/OmopTransformer/StandardProcedureConceptSelector.cs @@ -4,7 +4,7 @@ namespace OmopTransformer; [Description("Maps concepts to standard valid concepts in the `procedure` domain.")] -internal class StandardProcedureConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector +internal class StandardProcedureConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector { public object? GetValue() { diff --git a/OmopTransformer/Transformation/RecordTransformer.cs b/OmopTransformer/Transformation/RecordTransformer.cs index e17640b7..f6afbdf4 100644 --- a/OmopTransformer/Transformation/RecordTransformer.cs +++ b/OmopTransformer/Transformation/RecordTransformer.cs @@ -9,7 +9,7 @@ internal class RecordTransformer : IRecordTransformer { private readonly ILogger _logger; private readonly Opcs4Resolver _opcs4Resolver; - private readonly ConceptResolver _resolver; + private readonly StandardConceptResolver _resolver; private readonly Icdo3Resolver _icdo3Resolver; private readonly MeasurementMapsToValueResolver _relationshipResolver; private readonly Icd10NonStandardResolver _icd10NonStandardResolver; @@ -19,7 +19,7 @@ internal class RecordTransformer : IRecordTransformer public RecordTransformer( ILogger logger, - ConceptResolver resolver, + StandardConceptResolver resolver, Opcs4Resolver opcs4Resolver, Icdo3Resolver icdo3Resolver, MeasurementMapsToValueResolver relationshipResolver, @@ -56,6 +56,8 @@ public void PrintLogsAndResetLogger(ILoggerFactory loggerFactory) { _recordTransformLookupLogger.PrintLog(loggerFactory); _recordTransformLookupLogger.Reset(); + + _resolver.PrintLogsAndResetLogger(); } @@ -173,7 +175,7 @@ private void TransformSelector(IOmopRecord record, TransformAttribute tran .Concat(firstConstructorTypes.Any(type => type == typeof(Opcs4Resolver)) ? new[] { _opcs4Resolver } : new List()) .Concat(firstConstructorTypes.Any(type => type == typeof(Icdo3Resolver)) ? new[] { _icdo3Resolver } : new List()) .Concat(firstConstructorTypes.Any(type => type == typeof(SnomedResolver)) ? new[] { _snomedResolver } : new List()) - .Concat(firstConstructorTypes.Any(type => type == typeof(ConceptResolver)) ? new[] { _resolver } : new List()) + .Concat(firstConstructorTypes.Any(type => type == typeof(StandardConceptResolver)) ? new[] { _resolver } : new List()) .Concat(firstConstructorTypes.Any(type => type == typeof(Icd10NonStandardResolver)) ? new [] { _icd10NonStandardResolver } : new List()) .Concat(firstConstructorTypes.Any(type => type == typeof(Icd10StandardResolver)) ? new[] { _icd10StandardResolver } : new List()) .Concat(firstConstructorTypes.Any(type => type == typeof(MeasurementMapsToValueResolver)) ? new[] { _relationshipResolver } : new List()) diff --git a/OmopTransformer/Transformer.cs b/OmopTransformer/Transformer.cs index 69dcbe5c..4abafdcf 100644 --- a/OmopTransformer/Transformer.cs +++ b/OmopTransformer/Transformer.cs @@ -38,7 +38,7 @@ public async Task Transform( var computeStopwatch = new Stopwatch(); var getRecordsStopwatch = Stopwatch.StartNew(); var insertRecordsStopwatch = new Stopwatch(); - + var notEndOfRecords = true; int batchNumber = 0; diff --git a/docs/transformation-documentation/CosdV8ProcedureOccurrenceProcedureOpcs.json b/docs/transformation-documentation/CosdV8ProcedureOccurrenceProcedureOpcs.json index 17405937..9c651f41 100644 --- a/docs/transformation-documentation/CosdV8ProcedureOccurrenceProcedureOpcs.json +++ b/docs/transformation-documentation/CosdV8ProcedureOccurrenceProcedureOpcs.json @@ -17,7 +17,7 @@ ] } ], - "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n-- fail\n\t", + "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n ", "lookup_table_markdown": null }, { @@ -42,7 +42,7 @@ ] } ], - "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n-- fail\n\t", + "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n ", "lookup_table_markdown": null }, { @@ -60,7 +60,7 @@ ] } ], - "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n-- fail\n\t", + "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n ", "lookup_table_markdown": null }, { @@ -85,7 +85,7 @@ ] } ], - "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n-- fail\n\t", + "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n ", "lookup_table_markdown": null }, { @@ -103,7 +103,7 @@ ] } ], - "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n-- fail\n\t", + "query": "\nwith co as (\n select \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreLinkagePatientId.NHSNumber.@extension' as NhsNumber,\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureDate' as ProcedureDate,\n unnest(\n [\n [\n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS.@code'\n ], \n Record ->> '$.Colorectal.ColorectalCore.ColorectalCoreTreatment.ColorectalCoreSurgeryAndOtherProcedures.ProcedureOPCS[*].@code',\n ], recursive := true\n ) as ProcedureOpcsCode\n from omop_staging.cosd_staging_81\n where Type = 'CO'\n)\nselect\n distinct\n\t\tNhsNumber,\n\t\tProcedureDate,\n\t\tProcedureOpcsCode\nfrom co\nwhere co.ProcedureOpcsCode is not null;\n ", "lookup_table_markdown": null }, { diff --git a/docs/transformation-documentation/CosdV9DeathBasisOfDiagnosisCancer.json b/docs/transformation-documentation/CosdV9DeathBasisOfDiagnosisCancer.json index bf8cc1f5..9f08ec49 100644 --- a/docs/transformation-documentation/CosdV9DeathBasisOfDiagnosisCancer.json +++ b/docs/transformation-documentation/CosdV9DeathBasisOfDiagnosisCancer.json @@ -17,7 +17,7 @@ ] } ], - "query": "\n -- fail\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", + "query": "\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", "lookup_table_markdown": null }, { @@ -47,7 +47,7 @@ ] } ], - "query": "\n -- fail\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", + "query": "\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", "lookup_table_markdown": null }, { @@ -77,7 +77,7 @@ ] } ], - "query": "\n -- fail\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", + "query": "\n\t with cosddates as (\n select\n Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,\n cast(Record ->> '$.Treatment.TreatmentStartDateCancer' as date) as TreatmentStartDateCancer,\n cast(Record ->> '$.CancerCarePlan.MultidisciplinaryTeamDiscussionDateCancer' as date) as MultidisciplinaryTeamDiscussionDateCancer,\n cast(Record ->> '$.PrimaryPathway.Staging.StageDateFinalPretreatmentStage' as date) as StageDateFinalPretreatmentStage,\n cast(Record ->> '$.PrimaryPathway.LinkageDiagnosticDetails.DateOfPrimaryDiagnosisClinicallyAgreed' as date) as DateOfPrimaryDiagnosisClinicallyAgreed\n from omop_staging.cosd_staging_901\n where type = 'CO'\n and (Record ->> '$.PrimaryPathway.Diagnosis.BasisOfDiagnosisCancer.@code') in ('0', '1')\n), dates as (\n select NhsNumber, TreatmentStartDateCancer as \"Date\" from cosddates where TreatmentStartDateCancer is not null\n union\n select NhsNumber, MultidisciplinaryTeamDiscussionDateCancer as \"Date\" from cosddates where MultidisciplinaryTeamDiscussionDateCancer is not null\n union\n select NhsNumber, StageDateFinalPretreatmentStage as \"Date\" from cosddates where StageDateFinalPretreatmentStage is not null\n union\n select NhsNumber, DateOfPrimaryDiagnosisClinicallyAgreed as \"Date\" from cosddates where DateOfPrimaryDiagnosisClinicallyAgreed is not null\n)\nselect\n NhsNumber,\n make_date(cast(extract(year from max(\"Date\")) as integer), 12, 31) as DeathDate\nfrom dates\ngroup by NhsNumber;\n\t", "lookup_table_markdown": null }, { diff --git a/docs/transformation-documentation/Death_NhsNumber.md b/docs/transformation-documentation/Death_NhsNumber.md index cba71756..fe10e53d 100644 --- a/docs/transformation-documentation/Death_NhsNumber.md +++ b/docs/transformation-documentation/Death_NhsNumber.md @@ -141,7 +141,6 @@ where (Record ->> '$.Treatment.DischargeDestinationHospitalProviderSpell.@code') * `NhsNumber` Patient NHS Number [NHS NUMBER](https://www.datadictionary.nhs.uk/data_elements/nhs_number.html) ```sql --- fail with cosddates as ( select Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber, diff --git a/docs/transformation-documentation/Death_death_date.md b/docs/transformation-documentation/Death_death_date.md index c09d9d77..40de4a92 100644 --- a/docs/transformation-documentation/Death_death_date.md +++ b/docs/transformation-documentation/Death_death_date.md @@ -146,7 +146,6 @@ where (Record ->> '$.Treatment.DischargeDestinationHospitalProviderSpell.@code') * `DeathDate` The date on which a PERSON died or is officially deemed to have died. [MULTIDISCIPLINARY TEAM DISCUSSION DATE (CANCER)](https://www.datadictionary.nhs.uk/data_elements/multidisciplinary_team_discussion_date__cancer_.html), [TREATMENT START DATE (CANCER)](https://www.datadictionary.nhs.uk/data_elements/treatment_start_date__cancer_.html), [TNM STAGE GROUPING DATE (FINAL PRETREATMENT)](https://www.datadictionary.nhs.uk/data_elements/tnm_stage_grouping_date__final_pretreatment_.html), [DATE OF PRIMARY CANCER DIAGNOSIS (CLINICALLY AGREED)](https://www.datadictionary.nhs.uk/data_elements/date_of_primary_cancer_diagnosis__clinically_agreed_.html) ```sql --- fail with cosddates as ( select Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber, diff --git a/docs/transformation-documentation/Death_death_datetime.md b/docs/transformation-documentation/Death_death_datetime.md index 68ba2076..3a37fc36 100644 --- a/docs/transformation-documentation/Death_death_datetime.md +++ b/docs/transformation-documentation/Death_death_datetime.md @@ -85,7 +85,6 @@ where (Record ->> '$.Treatment.DischargeDestinationHospitalProviderSpell.@code') * `DeathDate` The date on which a PERSON died or is officially deemed to have died. [MULTIDISCIPLINARY TEAM DISCUSSION DATE (CANCER)](https://www.datadictionary.nhs.uk/data_elements/multidisciplinary_team_discussion_date__cancer_.html), [TREATMENT START DATE (CANCER)](https://www.datadictionary.nhs.uk/data_elements/treatment_start_date__cancer_.html), [TNM STAGE GROUPING DATE (FINAL PRETREATMENT)](https://www.datadictionary.nhs.uk/data_elements/tnm_stage_grouping_date__final_pretreatment_.html), [DATE OF PRIMARY CANCER DIAGNOSIS (CLINICALLY AGREED)](https://www.datadictionary.nhs.uk/data_elements/date_of_primary_cancer_diagnosis__clinically_agreed_.html) ```sql --- fail with cosddates as ( select Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber, diff --git a/docs/transformation-documentation/ProcedureOccurrence_nhs_number.md b/docs/transformation-documentation/ProcedureOccurrence_nhs_number.md index 452f7a52..82e2ef9f 100644 --- a/docs/transformation-documentation/ProcedureOccurrence_nhs_number.md +++ b/docs/transformation-documentation/ProcedureOccurrence_nhs_number.md @@ -476,8 +476,6 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - ``` diff --git a/docs/transformation-documentation/ProcedureOccurrence_procedure_date.md b/docs/transformation-documentation/ProcedureOccurrence_procedure_date.md index 42de09cd..b5502a64 100644 --- a/docs/transformation-documentation/ProcedureOccurrence_procedure_date.md +++ b/docs/transformation-documentation/ProcedureOccurrence_procedure_date.md @@ -489,8 +489,6 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - ``` diff --git a/docs/transformation-documentation/ProcedureOccurrence_procedure_datetime.md b/docs/transformation-documentation/ProcedureOccurrence_procedure_datetime.md index 26eb0674..12de89d3 100644 --- a/docs/transformation-documentation/ProcedureOccurrence_procedure_datetime.md +++ b/docs/transformation-documentation/ProcedureOccurrence_procedure_datetime.md @@ -493,8 +493,6 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - ``` diff --git a/docs/transformation-documentation/ProcedureOccurrence_procedure_source_concept_id.md b/docs/transformation-documentation/ProcedureOccurrence_procedure_source_concept_id.md index 26151147..80037a8b 100644 --- a/docs/transformation-documentation/ProcedureOccurrence_procedure_source_concept_id.md +++ b/docs/transformation-documentation/ProcedureOccurrence_procedure_source_concept_id.md @@ -577,8 +577,6 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - ``` diff --git a/docs/transformation-documentation/ProcedureOccurrence_procedure_source_value.md b/docs/transformation-documentation/ProcedureOccurrence_procedure_source_value.md index 3cbbb404..f4226dbb 100644 --- a/docs/transformation-documentation/ProcedureOccurrence_procedure_source_value.md +++ b/docs/transformation-documentation/ProcedureOccurrence_procedure_source_value.md @@ -481,8 +481,6 @@ select ProcedureOpcsCode from co where co.ProcedureOpcsCode is not null; --- fail - ```