Map unknown concepts to 0 rather than null. When mapping from known c…

OmopTransformer/COSD/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml

@@ -1,6 +1,5 @@
 <Query>
     <Sql>
-        -- fail
 	    with cosddates as (
     select
         Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,

OmopTransformer/COSD/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml

@@ -22,8 +22,7 @@ select
 		ProcedureOpcsCode
 from co
 where co.ProcedureOpcsCode is not null;
--- fail
-	</Sql>
+    </Sql>
 	<Explanations>
 		<Explanation columnName="NhsNumber">
 			<Description>Patient NHS Number</Description>

OmopTransformer/ConceptDeviceSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Finds related devices for a concept.")]
-internal class ConceptDeviceSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class ConceptDeviceSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/ConceptLookup.cs

@@ -41,16 +41,21 @@ private Dictionary<string, int> GetCodes()
 
     public virtual string FormatCode(string code) => code;
 
-    public int? GetConceptCode(string? code)
+    public int GetConceptCode(string? code)
     {
-        lock (_loadingLock)
+        const int unknownConceptId = 0;
+
+        if (_mappings == null)
         {
-            _mappings ??= GetCodes();
+            lock (_loadingLock)
+            {
+                _mappings ??= GetCodes();
+            }
         }
 
         if (code == null)
         {
-            return null;
+            return unknownConceptId;
         }
 
         var formatCode = FormatCode(code);
@@ -70,7 +75,7 @@ private Dictionary<string, int> GetCodes()
             }
         }
 
-        return null;
+        return unknownConceptId;
     }
 }
 

OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs

@@ -122,6 +122,7 @@ where r.RecordConnectionIdentifier is not null
         and co.person_id = p.person_id
         and co.RecordConnectionIdentifier = r.RecordConnectionIdentifier
         and co.condition_concept_id = r.condition_concept_id
+        and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value)
 )
 and not exists (
     select 1 
@@ -130,6 +131,7 @@ where r.RecordConnectionIdentifier is null
         and co.condition_concept_id = r.condition_concept_id
         and co.condition_start_date = r.condition_start_date
         and co.person_id = p.person_id
+        and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value)
 );
 
 

OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs

@@ -164,6 +164,7 @@ from cdm.device_exposure vo
                 and vo.device_concept_id = r.device_concept_id
                 and vo.device_exposure_start_date = r.device_exposure_start_date
                 and vo.device_exposure_end_date = r.device_exposure_end_date
+                and (vo.device_concept_id != 0 or ((vo.device_source_value is null and r.device_source_value is null) or vo.device_source_value = r.device_source_value))
         )
     );
 

OmopTransformer/Omop/DrugExposure/DrugExposureRecorder.cs

@@ -156,6 +156,7 @@ from cdm.drug_exposure vo
             where vo.RecordConnectionIdentifier = r.RecordConnectionIdentifier
                 and vo.person_id = p.person_id
                 and vo.drug_concept_id = r.drug_concept_id
+
         )
     )
     or
@@ -167,6 +168,7 @@ from cdm.drug_exposure vo
             where vo.person_id = p.person_id
                 and vo.drug_concept_id = r.drug_concept_id
                 and vo.drug_exposure_start_date = r.drug_exposure_start_date
+                and (vo.drug_concept_id != 0 or ((vo.drug_source_value is null and r.drug_source_value is null) or vo.drug_source_value = r.drug_source_value))
         )
     );
 

OmopTransformer/Omop/Measurement/MeasurementRecorder.cs

@@ -143,6 +143,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.RecordConnectionIdentifier is null
             and r.HospitalProviderSpellNumber is null
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     )
     and not exists (
         select 1
@@ -153,6 +154,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.RecordConnectionIdentifier is not null
             and m.RecordConnectionIdentifier = r.RecordConnectionIdentifier
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     )
     and not exists (
         select 1
@@ -163,6 +165,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.HospitalProviderSpellNumber is not null
             and m.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     );
 
 truncate table omop_staging.measurement_row;",

OmopTransformer/Omop/Observation/ObservationRecorder.cs

@@ -157,15 +157,17 @@ r.RecordConnectionIdentifier is not null and
                 o.observation_concept_id = r.observation_concept_id and
                 o.RecordConnectionIdentifier = r.RecordConnectionIdentifier and
                 (r.HospitalProviderSpellNumber is null or o.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber) and
-                (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id)
+                (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id) and
+                (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string))
             )
             or
             (
                 r.RecordConnectionIdentifier is null and
                 o.person_id = p.person_id and
                 o.observation_date = r.observation_date and
                 o.observation_concept_id = r.observation_concept_id and
-                o.observation_source_concept_id = r.observation_source_concept_id
+                o.observation_source_concept_id = r.observation_source_concept_id and
+                (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string))
             )
     );
 

OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs

@@ -128,14 +128,17 @@ r.RecordConnectionIdentifier is not null and
                 vo.RecordConnectionIdentifier = r.RecordConnectionIdentifier and
                 vo.person_id = p.person_id and
                 vo.procedure_date = r.procedure_date and
-                vo.procedure_concept_id = r.procedure_concept_id
+                vo.procedure_concept_id = r.procedure_concept_id and     
+                (vo.procedure_concept_id != 0 or ((vo.procedure_source_value is null and r.procedure_source_value is null) or vo.procedure_source_value = r.procedure_source_value))
+
             )
             or
             (
                 r.RecordConnectionIdentifier is null and
                 vo.procedure_date = r.procedure_date and
                 vo.procedure_concept_id = r.procedure_concept_id and
-                vo.person_id = p.person_id
+                vo.person_id = p.person_id and
+                (vo.procedure_concept_id != 0 or ((vo.procedure_source_value is null and r.procedure_source_value is null) or vo.procedure_source_value = r.procedure_source_value))
             )
     );
 

OmopTransformer/OxfordGP/OxfordGPTransformer.cs

@@ -27,7 +27,7 @@ internal class OxfordGPTransformer : Transformer
     private readonly ILocationRecorder _locationRecorder;
     private readonly IPersonRecorder _personRecorder;
     private readonly IDeathRecorder _deathRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IConditionOccurrenceRecorder _conditionOccurrenceRecorder;
     private readonly IVisitOccurrenceRecorder _visitOccurrenceRecorder;
@@ -42,7 +42,7 @@ public OxfordGPTransformer(
         ILocationRecorder locationRecorder,
         IPersonRecorder personRecorder,
         IDeathRecorder deathRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder,
@@ -126,7 +126,5 @@ await Transform<OxfordGPDeviceExposureRecord, OxfordGPDeviceExposure>(
             "Oxford GP Device Exposure",
             runId,
             cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/OxfordLab/OxfordLabTransformer.cs

@@ -10,7 +10,7 @@ namespace OmopTransformer.OxfordLab;
 
 internal class OxfordLabTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IMeasurementRecorder _measurementRecorder;
     private readonly IObservationRecorder _observationRecorder;
 
@@ -20,7 +20,7 @@ public OxfordLabTransformer(
         IRecordProvider recordProvider,
         IMeasurementRecorder measurementRecorder,
         IObservationRecorder observationRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory) : base(recordTransformer,
         transformOptions,
@@ -49,7 +49,5 @@ await Transform<OxfordLabGeneralCommentRecord, OxfordLabGeneralComment>(
             "Oxford Lab General Comments",
             runId,
             cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs

@@ -9,15 +9,15 @@ namespace OmopTransformer.OxfordPrescribing;
 
 internal class OxfordPrescribingTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IDrugExposureRecorder _drugExposureRecorder;
 
     public OxfordPrescribingTransformer(
         IRecordTransformer recordTransformer,
         TransformOptions transformOptions,
         IRecordProvider recordProvider,
         IDrugExposureRecorder drugExposureRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory) : base(recordTransformer,
         transformOptions,
@@ -45,7 +45,5 @@ await Transform<OxfordPrescribingDrugExposureRecord, OxfordPrescribingDrugExposu
           "Oxford Prescribing Drugs",
           runId,
           cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/OxfordSpineDeath/OxfordSpineDeathTransformer.cs

@@ -8,14 +8,14 @@ namespace OmopTransformer.OxfordSpineDeath;
 
 internal class OxfordSpineDeathTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IDeathRecorder _deathRecorder;
 
     public OxfordSpineDeathTransformer(
         IRecordTransformer recordTransformer,
         TransformOptions transformOptions,
         IRecordProvider recordProvider,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory, IDeathRecorder deathRecorder) : base(recordTransformer,
         transformOptions,
@@ -37,7 +37,5 @@ public async Task Transform(CancellationToken cancellationToken)
             "Oxford Spine Death",
             runId,
             cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/Program.cs

@@ -361,7 +361,7 @@ private static async Task Main(string[] args)
         builder.Services.AddSingleton<Opcs4Resolver>();
         builder.Services.AddSingleton<Icdo3Resolver>();
         builder.Services.AddSingleton<SnomedResolver>();
-        builder.Services.AddSingleton<ConceptResolver>();
+        builder.Services.AddSingleton<StandardConceptResolver>();
         builder.Services.AddSingleton<MeasurementMapsToValueResolver>();
 
         IHostEnvironment env = builder.Environment;

OmopTransformer/SUS/AE/SusAETransformer.cs

@@ -37,7 +37,7 @@ internal class SusAETransformer : Transformer
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
 
     public SusAETransformer(
@@ -52,7 +52,7 @@ public SusAETransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder, 
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IDeviceExposureRecorder deviceExposureRecorder,
         IObservationRecorder observationRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,
@@ -157,7 +157,5 @@ await Transform<SusAECareSiteRecord, SusAECareSite>(
            "SUS AE CareSite",
            runId,
            cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/SUS/APC/SusAPCTransformer.cs

@@ -52,7 +52,7 @@ internal class SusAPCTransformer : Transformer
     private readonly IDeathRecorder _deathRecorder;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
     private readonly IProviderRecorder _providerRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
@@ -71,7 +71,7 @@ public SusAPCTransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IDeviceExposureRecorder deviceExposureRecorder,
         IObservationRecorder observationRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,
@@ -258,7 +258,5 @@ await Transform<SusAPCReferralReceivedDateForInpatientsRecord, SusAPCReferralRec
            "SUS APC ReferralReceivedDateForInpatients",
            runId,
            cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }

OmopTransformer/SUS/OP/SusOPTransformer.cs

@@ -40,7 +40,7 @@ internal class SusOPTransformer : Transformer
     private readonly IDeathRecorder _deathRecorder;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
     private readonly IProviderRecorder _providerRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
@@ -59,7 +59,7 @@ public SusOPTransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder, 
-        ConceptResolver conceptResolver, 
+        StandardConceptResolver conceptResolver, 
         IObservationRecorder observationRecorder,
         IDeviceExposureRecorder deviceExposureRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,
@@ -172,7 +172,5 @@ await Transform<SusOPDeviceExposureRecord, SusOPDeviceExposure>(
            "SUS OP DeviceExposure",
            runId,
            cancellationToken);
-
-        _conceptResolver.PrintErrors();
     }
 }