Map unknown concepts to 0 rather than null. When mapping from known c…

OmopTransformer/COSD/Colorectal/ProcedureOccurrence/CosdV8ProcedureOccurrenceProcedureOpcs/CosdV8ProcedureOccurrenceProcedureOpcs.xml

@@ -22,8 +22,7 @@ select
 		ProcedureOpcsCode
 from co
 where co.ProcedureOpcsCode is not null;
--- fail
-	</Sql>
+    </Sql>
 	<Explanations>
 		<Explanation columnName="NhsNumber">
 			<Description>Patient NHS Number</Description>

OmopTransformer/COSD/Core/Death/v9DeathBasisOfDiagnosisCancer/CosdV9BasisOfDiagnosisCancer.xml

@@ -1,6 +1,5 @@
 <Query>
     <Sql>
-        -- fail
 	    with cosddates as (
     select
         Record ->> '$.LinkagePatientId.NhsNumber.@extension' as NhsNumber,

OmopTransformer/ConceptDeviceSelector.cs

@@ -4,7 +4,7 @@
 namespace OmopTransformer;
 
 [Description("Finds related devices for a concept.")]
-internal class ConceptDeviceSelector(int? conceptId, ConceptResolver resolver) : ISelector
+internal class ConceptDeviceSelector(int? conceptId, StandardConceptResolver resolver) : ISelector
 {
     public object? GetValue()
     {

OmopTransformer/ConceptLookup.cs

@@ -41,16 +41,21 @@ private Dictionary<string, int> GetCodes()
 
     public virtual string FormatCode(string code) => code;
 
-    public int? GetConceptCode(string? code)
+    public int GetConceptCode(string? code)
     {
-        lock (_loadingLock)
+        const int unknownConceptId = 0;
+
+        if (_mappings == null)
         {
-            _mappings ??= GetCodes();
+            lock (_loadingLock)
+            {
+                _mappings ??= GetCodes();
+            }
         }
 
         if (code == null)
         {
-            return null;
+            return unknownConceptId;
         }
 
         var formatCode = FormatCode(code);
@@ -70,7 +75,7 @@ private Dictionary<string, int> GetCodes()
             }
         }
 
-        return null;
+        return unknownConceptId;
     }
 }
 

OmopTransformer/Init/Initiation.cs

@@ -635,16 +635,13 @@ invalid_count integer NOT NULL
 CREATE TABLE omop_staging.concept_code_map(
 	source_concept_code varchar(50) NOT NULL,
 	source_concept_id integer NOT NULL,
-	target_concept_id integer NOT NULL,
-	domain_id varchar(50) NOT NULL,
-	mapped_from_standard bit NOT NULL,
- constraint PK_omop_staging_concept_code_map_source_concept_id_target_concept_id PRIMARY KEY 
-	(
-		source_concept_id,
-		target_concept_id 
-	)
+	target_concept_id integer NULL,
+    source_domain_id varchar(50) NOT NULL,
+	target_domain_id varchar(50) NULL,
+	mapped_from_standard tinyint NOT NULL
 );
 
+
 CREATE TABLE omop_staging.cosd_staging_81(
 	SubmissionName varchar(200) NOT NULL,
 	FileName varchar(200) NOT NULL,
@@ -2530,15 +2527,17 @@ insert into omop_staging.concept_code_map
  	source_concept_code,
 	source_concept_id,
  	target_concept_id,
- 	domain_id,
+ 	source_domain_id,
+  target_domain_id,
  	mapped_from_standard
  )
  with InvalidConceptMap as (
  	select
  		c1.concept_code as source_concept_code,
 		c1.concept_id as source_concept_id,
 		c2.concept_id as target_concept_id,
-		c2.domain_id
+    c1.domain_id as source_domain_id,
+		c2.domain_id as target_domain_id
  	from cdm.concept c1
 		inner join cdm.concept_relationship cr
 			on c1.concept_id = cr.concept_id_1
@@ -2560,7 +2559,8 @@ and current_date between c2.valid_start_date and c2.valid_end_date
  		c.concept_code as source_concept_code,
 		c.concept_id as source_concept_id,
  		c.concept_id as target_concept_id,
- 		c.domain_id as domain_id,
+ 		c.domain_id as target_domain_id,
+    c.domain_id as source_domain_id,
  		1 as mapped_from_standard
  	from cdm.concept c
  	where c.standard_concept is not null
@@ -2573,7 +2573,8 @@ and current_date between c.valid_start_date and c.valid_end_date
  		icm.source_concept_code,
 		icm.source_concept_id,
  		icm.target_concept_id as concept_id,
- 		c.domain_id,
+ 		icm.source_domain_id,
+		icm.target_domain_id, 
  		0 as mapped_from_standard
  	from InvalidConceptMap icm
  		inner join cdm.concept c
@@ -2586,7 +2587,8 @@ where icm.target_concept_id is not null
 		source_concept_code,
 		source_concept_id,
  		target_concept_id,
- 		domain_id,
+ 		source_domain_id,
+    target_domain_id,
  		mapped_from_standard
 from Mapped;
 
@@ -2595,14 +2597,16 @@ insert into omop_staging.concept_code_map
 	source_concept_code,
 	source_concept_id,
 	target_concept_id,
-	domain_id,
+	source_domain_id,
+  target_domain_id,
 	mapped_from_standard
 )
 select
 	c.concept_code as source_concept_code,
 	c.concept_id as source_concept_id,
 	rxnormConcept.concept_id as target_concept_id,
-	'Drug' as domain_id, 
+	'Drug' as source_domain_id, 
+  'Drug' as target_domain_id,
 	1 as mapped_from_standard
 from cdm.concept c
 	inner join cdm.concept dmdConcept 
@@ -2625,6 +2629,30 @@ from omop_staging.concept_code_map ccm
 	and rxnormConcept.standard_concept = 'S'
 	and rxnormConcept.domain_id = 'Drug'
 	and rxnormConcept.vocabulary_id in ('RxNorm', 'RxNorm Extension');
+
+insert into omop_staging.concept_code_map
+(
+  source_concept_code,
+  source_concept_id,
+  target_concept_id,
+  source_domain_id,
+  target_domain_id,
+  mapped_from_standard
+)
+select
+    c.concept_code,
+    c.concept_id,
+    null,
+    c.domain_id,
+    null,
+    0
+from cdm.concept c
+where c.standard_concept is null
+  and not exists (
+    select *
+    from omop_staging.concept_code_map ccm
+    where ccm.source_concept_id = c.concept_id
+  )
 ");
 
 

OmopTransformer/Omop/ConditionOccurrence/ConditionOccurrenceRecorder.cs

@@ -123,6 +123,7 @@ where r.RecordConnectionIdentifier is not null
         and co.person_id = p.person_id
         and co.RecordConnectionIdentifier = r.RecordConnectionIdentifier
         and co.condition_concept_id = r.condition_concept_id
+        and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value)
 )
 and not exists (
     select 1 
@@ -131,6 +132,7 @@ where r.RecordConnectionIdentifier is null
         and co.condition_concept_id = r.condition_concept_id
         and co.condition_start_date = r.condition_start_date
         and co.person_id = p.person_id
+        and (co.condition_concept_id != 0 or co.condition_source_value = r.condition_source_value)
 );
 
 

OmopTransformer/Omop/DeviceExposure/DeviceExposureRecorder.cs

@@ -165,6 +165,7 @@ from cdm.device_exposure vo
                 and vo.device_concept_id = r.device_concept_id
                 and vo.device_exposure_start_date = r.device_exposure_start_date
                 and vo.device_exposure_end_date = r.device_exposure_end_date
+                and (vo.device_concept_id != 0 or ((vo.device_source_value is null and r.device_source_value is null) or vo.device_source_value = r.device_source_value))
         )
     );