Map unknown concepts to 0 rather than null. When mapping from known c…

OmopTransformer/ConceptDeviceSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Finds related devices for a concept.")]
-internal class ConceptDeviceSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class ConceptDeviceSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/ConceptLookup.cs

@@ -41,16 +41,21 @@ private Dictionary<string, int> GetCodes()
 
     public virtual string FormatCode(string code) => code;
 
-    public int? GetConceptCode(string? code)
+    public int GetConceptCode(string? code)
     {
-        lock (_loadingLock)
+        const int unknownConceptId = 0;
+
+        if (_mappings == null)
         {
-            _mappings ??= GetCodes();
+            lock (_loadingLock)
+            {
+                _mappings ??= GetCodes();
+            }
         }
 
         if (code == null)
         {
-            return null;
+            return unknownConceptId;
         }
 
         var formatCode = FormatCode(code);
@@ -70,7 +75,7 @@ private Dictionary<string, int> GetCodes()
             }
         }
 
-        return null;
+        return unknownConceptId;
     }
 }
 

OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs

@@ -122,6 +122,7 @@ where r.RecordConnectionIdentifier is not null
         and co.person_id = p.person_id
         and co.RecordConnectionIdentifier = r.RecordConnectionIdentifier
         and co.condition_concept_id = r.condition_concept_id
+        and (co.condition_concept_id != 0 or and co.condition_source_value = r.condition_source_value)
 )
 and not exists (
     select 1 
@@ -130,6 +131,7 @@ where r.RecordConnectionIdentifier is null
         and co.condition_concept_id = r.condition_concept_id
         and co.condition_start_date = r.condition_start_date
         and co.person_id = p.person_id
+        and (co.condition_concept_id != 0 or and co.condition_source_value = r.condition_source_value)
 );
 
 

OmopTransformer/Omop/Measurement/MeasurementRecorder.cs

@@ -143,6 +143,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.RecordConnectionIdentifier is null
             and r.HospitalProviderSpellNumber is null
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     )
     and not exists (
         select 1
@@ -153,6 +154,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.RecordConnectionIdentifier is not null
             and m.RecordConnectionIdentifier = r.RecordConnectionIdentifier
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     )
     and not exists (
         select 1
@@ -163,6 +165,7 @@ from cdm.measurement m
             and (r.measurement_source_concept_id is null or m.measurement_source_concept_id = r.measurement_source_concept_id)
             and r.HospitalProviderSpellNumber is not null
             and m.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber
+            and (m.measurement_concept_id != 0 or ((m.value_source_value is null and r.value_source_value is null) or m.value_source_value = r.value_source_value))
     );
 
 truncate table omop_staging.measurement_row;",

OmopTransformer/Omop/Observation/ObservationRecorder.cs

@@ -157,15 +157,17 @@ r.RecordConnectionIdentifier is not null and
                 o.observation_concept_id = r.observation_concept_id and
                 o.RecordConnectionIdentifier = r.RecordConnectionIdentifier and
                 (r.HospitalProviderSpellNumber is null or o.HospitalProviderSpellNumber = r.HospitalProviderSpellNumber) and
-                (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id)
+                (r.observation_source_concept_id is null or o.observation_source_concept_id = r.observation_source_concept_id) and
+                (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string))
             )
             or
             (
                 r.RecordConnectionIdentifier is null and
                 o.person_id = p.person_id and
                 o.observation_date = r.observation_date and
                 o.observation_concept_id = r.observation_concept_id and
-                o.observation_source_concept_id = r.observation_source_concept_id
+                o.observation_source_concept_id = r.observation_source_concept_id and
+                (o.observation_concept_id != 0 or ((o.value_as_string is null and r.value_as_string is null) or o.value_as_string = r.value_as_string))
             )
     );
 

OmopTransformer/Omop/ProcedureOccurrence/ProcedureOccurrenceRecorder.cs

@@ -128,7 +128,9 @@ r.RecordConnectionIdentifier is not null and
                 vo.RecordConnectionIdentifier = r.RecordConnectionIdentifier and
                 vo.person_id = p.person_id and
                 vo.procedure_date = r.procedure_date and
-                vo.procedure_concept_id = r.procedure_concept_id
+                vo.procedure_concept_id = r.procedure_concept_id and     
+                (o.procedure_concept_id != 0 or ((o.procedure_source_value is null and r.procedure_source_value is null) or o.procedure_source_value = r.procedure_source_value))
+
             )
             or
             (

OmopTransformer/OxfordGP/OxfordGPTransformer.cs

@@ -27,7 +27,7 @@ internal class OxfordGPTransformer : Transformer
     private readonly ILocationRecorder _locationRecorder;
     private readonly IPersonRecorder _personRecorder;
     private readonly IDeathRecorder _deathRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IConditionOccurrenceRecorder _conditionOccurrenceRecorder;
     private readonly IVisitOccurrenceRecorder _visitOccurrenceRecorder;
@@ -42,7 +42,7 @@ public OxfordGPTransformer(
         ILocationRecorder locationRecorder,
         IPersonRecorder personRecorder,
         IDeathRecorder deathRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder,

OmopTransformer/OxfordLab/OxfordLabTransformer.cs

@@ -10,7 +10,7 @@ namespace OmopTransformer.OxfordLab;
 
 internal class OxfordLabTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IMeasurementRecorder _measurementRecorder;
     private readonly IObservationRecorder _observationRecorder;
 
@@ -20,7 +20,7 @@ public OxfordLabTransformer(
         IRecordProvider recordProvider,
         IMeasurementRecorder measurementRecorder,
         IObservationRecorder observationRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory) : base(recordTransformer,
         transformOptions,

OmopTransformer/OxfordPrescribing/OxfordPrescribingTransformer.cs

@@ -9,15 +9,15 @@ namespace OmopTransformer.OxfordPrescribing;
 
 internal class OxfordPrescribingTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IDrugExposureRecorder _drugExposureRecorder;
 
     public OxfordPrescribingTransformer(
         IRecordTransformer recordTransformer,
         TransformOptions transformOptions,
         IRecordProvider recordProvider,
         IDrugExposureRecorder drugExposureRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory) : base(recordTransformer,
         transformOptions,

OmopTransformer/OxfordSpineDeath/OxfordSpineDeathTransformer.cs

@@ -8,14 +8,14 @@ namespace OmopTransformer.OxfordSpineDeath;
 
 internal class OxfordSpineDeathTransformer : Transformer
 {
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly IDeathRecorder _deathRecorder;
 
     public OxfordSpineDeathTransformer(
         IRecordTransformer recordTransformer,
         TransformOptions transformOptions,
         IRecordProvider recordProvider,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IRunAnalysisRecorder runAnalysisRecorder,
         ILoggerFactory loggerFactory, IDeathRecorder deathRecorder) : base(recordTransformer,
         transformOptions,

OmopTransformer/Program.cs

@@ -361,7 +361,7 @@ private static async Task Main(string[] args)
         builder.Services.AddSingleton<Opcs4Resolver>();
         builder.Services.AddSingleton<Icdo3Resolver>();
         builder.Services.AddSingleton<SnomedResolver>();
-        builder.Services.AddSingleton<ConceptResolver>();
+        builder.Services.AddSingleton<StandardConceptResolver>();
         builder.Services.AddSingleton<MeasurementMapsToValueResolver>();
 
         IHostEnvironment env = builder.Environment;

OmopTransformer/SUS/AE/SusAETransformer.cs

@@ -37,7 +37,7 @@ internal class SusAETransformer : Transformer
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
 
     public SusAETransformer(
@@ -52,7 +52,7 @@ public SusAETransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder, 
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IDeviceExposureRecorder deviceExposureRecorder,
         IObservationRecorder observationRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,

OmopTransformer/SUS/APC/SusAPCTransformer.cs

@@ -52,7 +52,7 @@ internal class SusAPCTransformer : Transformer
     private readonly IDeathRecorder _deathRecorder;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
     private readonly IProviderRecorder _providerRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
@@ -71,7 +71,7 @@ public SusAPCTransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder,
-        ConceptResolver conceptResolver,
+        StandardConceptResolver conceptResolver,
         IDeviceExposureRecorder deviceExposureRecorder,
         IObservationRecorder observationRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,

OmopTransformer/SUS/OP/SusOPTransformer.cs

@@ -40,7 +40,7 @@ internal class SusOPTransformer : Transformer
     private readonly IDeathRecorder _deathRecorder;
     private readonly IProcedureOccurrenceRecorder _procedureOccurrenceRecorder;
     private readonly IObservationRecorder _observationRecorder;
-    private readonly ConceptResolver _conceptResolver;
+    private readonly StandardConceptResolver _conceptResolver;
     private readonly ICareSiteRecorder _careSiteRecorder;
     private readonly IProviderRecorder _providerRecorder;
     private readonly IDeviceExposureRecorder _deviceExposureRecorder;
@@ -59,7 +59,7 @@ public SusOPTransformer(
         IVisitDetailRecorder visitDetailRecorder,
         IDeathRecorder deathRecorder,
         IProcedureOccurrenceRecorder procedureOccurrenceRecorder, 
-        ConceptResolver conceptResolver, 
+        StandardConceptResolver conceptResolver, 
         IObservationRecorder observationRecorder,
         IDeviceExposureRecorder deviceExposureRecorder,
         IRunAnalysisRecorder runAnalysisRecorder,

OmopTransformer/ConceptResolver.cs → OmopTransformer/StandardConceptResolver.cs

@@ -5,10 +5,10 @@
 
 namespace OmopTransformer;
 
-internal class ConceptResolver
+internal class StandardConceptResolver
 {
     private readonly Configuration _configuration;
-    private readonly ILogger<ConceptResolver> _logger;
+    private readonly ILogger<StandardConceptResolver> _logger;
 
     private Dictionary<int, IGrouping<int, Row>>? _mappings;
     private Dictionary<int, IReadOnlyCollection<int>>? _devicesByConceptId;
@@ -18,7 +18,7 @@ internal class ConceptResolver
     private readonly Dictionary<int, int> _unableToMapByFrequency = new();
     private readonly object _unableToMapByFrequencyLock = new();
 
-    public ConceptResolver(IOptions<Configuration> configuration, ILogger<ConceptResolver> logger)
+    public StandardConceptResolver(IOptions<Configuration> configuration, ILogger<StandardConceptResolver> logger)
     {
         _logger = logger;
         _configuration = configuration.Value;
@@ -96,15 +96,23 @@ private void EnsureDeviceMapping()
 
     public int[] GetConcepts(int conceptId, string? domain)
     {
+        const int unknownConceptId = 0;
+
         EnsureMapping();
+        bool foundMapping = false;
 
         if (_mappings!.TryGetValue(conceptId, out var value))
         {
-            return
+            foundMapping = true;
+
+            var results =
                 value
                     .Where(row => domain == null || row.domain_id!.Equals(domain, StringComparison.OrdinalIgnoreCase))
                     .Select(row => row.target_concept_id)
                     .ToArray();
+
+            if (results.Length > 0)
+                return results;
         }
 
         lock (_unableToMapByFrequencyLock)
@@ -119,6 +127,15 @@ public int[] GetConcepts(int conceptId, string? domain)
             }
         }
 
+        // if no domain specified, return unknown concept
+        // if domain specified, and mapping not found, return unknown concept
+        // if domain specified, and mapping found but it is in the wrong domain, return empty 
+
+        if (domain == null || foundMapping == false)
+        {
+            return new int[] { unknownConceptId };
+        }
+
         return new int[] { };
     }
 

OmopTransformer/StandardConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Converts concepts to standard concepts.")]
-internal class StandardConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/StandardConditionConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Maps concepts to standard valid concepts in the `condition` domain.")]
-internal class StandardConditionConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardConditionConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/StandardDeviceConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Maps concepts to standard valid concepts in the `device` domain.")]
-internal class StandardDeviceConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardDeviceConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/StandardDrugConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Maps concepts to standard valid concepts in the `drug` domain.")]
-internal class StandardDrugConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardDrugConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/StandardMeasurementConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Maps concepts to standard valid concepts in the `measurement` domain.")]
-internal class StandardMeasurementConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardMeasurementConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/StandardProcedureConceptSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Maps concepts to standard valid concepts in the `procedure` domain.")]
-internal class StandardProcedureConceptSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class StandardProcedureConceptSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/Transformation/RecordTransformer.cs

@@ -9,7 +9,7 @@ internal class RecordTransformer : IRecordTransformer
 {
     private readonly ILogger<RecordTransformer> _logger;
     private readonly Opcs4Resolver _opcs4Resolver;
-    private readonly ConceptResolver _resolver;
+    private readonly StandardConceptResolver _resolver;
     private readonly Icdo3Resolver _icdo3Resolver;
     private readonly MeasurementMapsToValueResolver _relationshipResolver;
     private readonly Icd10NonStandardResolver _icd10NonStandardResolver;
@@ -19,7 +19,7 @@ internal class RecordTransformer : IRecordTransformer
 
     public RecordTransformer(
         ILogger<RecordTransformer> logger, 
-        ConceptResolver resolver, 
+        StandardConceptResolver resolver, 
         Opcs4Resolver opcs4Resolver, 
         Icdo3Resolver icdo3Resolver,
         MeasurementMapsToValueResolver relationshipResolver, 
@@ -173,7 +173,7 @@ private void TransformSelector<T>(IOmopRecord<T> record, TransformAttribute tran
                 .Concat(firstConstructorTypes.Any(type => type == typeof(Opcs4Resolver)) ? new[] { _opcs4Resolver } : new List<object>())
                 .Concat(firstConstructorTypes.Any(type => type == typeof(Icdo3Resolver)) ? new[] { _icdo3Resolver } : new List<object>())
                 .Concat(firstConstructorTypes.Any(type => type == typeof(SnomedResolver)) ? new[] { _snomedResolver } : new List<object>())
-                .Concat(firstConstructorTypes.Any(type => type == typeof(ConceptResolver)) ? new[] { _resolver } : new List<object>())
+                .Concat(firstConstructorTypes.Any(type => type == typeof(StandardConceptResolver)) ? new[] { _resolver } : new List<object>())
                 .Concat(firstConstructorTypes.Any(type => type == typeof(Icd10NonStandardResolver)) ? new [] { _icd10NonStandardResolver } :  new List<object>())
                 .Concat(firstConstructorTypes.Any(type => type == typeof(Icd10StandardResolver)) ? new[] { _icd10StandardResolver } :  new List<object>())
                 .Concat(firstConstructorTypes.Any(type => type == typeof(MeasurementMapsToValueResolver)) ? new[] { _relationshipResolver } : new List<object>())