Skip to content

Commit aa5d000

Browse files
james-cockaynejames-cockayne
authored
Merge pull request #271 from answerdigital/feature/faster-finalise
Optimise finalise scripts
2 parents 6bbe8f4 + e21b69d commit aa5d000

File tree

1 file changed

+71
-52
lines changed

1 file changed

+71
-52
lines changed

‎OmopTransformer/Omop/Prune/OmopFinaliser.cs‎

Lines changed: 71 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -472,14 +472,17 @@ await connection.ExecuteAsync(@"
472472
473473
-- Combined DELETE statements for the 'death' table
474474
-- Deletes death records where related events occur >60 days after death
475-
DELETE FROM cdm.death
476-
WHERE person_id IN (
477-
SELECT person_id FROM cdm.DEVICE_EXPOSURE
478-
WHERE
479-
(DEVICE_EXPOSURE_END_DATE IS NOT NULL AND DEVICE_EXPOSURE_END_DATE > (death.death_date + INTERVAL '60' DAY)) OR
480-
(DEVICE_EXPOSURE_START_DATE IS NOT NULL AND DEVICE_EXPOSURE_START_DATE > (death.death_date + INTERVAL '60' DAY)) OR
481-
(DEVICE_EXPOSURE_END_DATETIME IS NOT NULL AND CAST(DEVICE_EXPOSURE_END_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY)) OR
482-
(DEVICE_EXPOSURE_START_DATETIME IS NOT NULL AND CAST(DEVICE_EXPOSURE_START_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
475+
delete from cdm.death d
476+
where exists (
477+
select 1
478+
from cdm.DEVICE_EXPOSURE de
479+
where de.person_id = d.person_id
480+
and (
481+
(de.DEVICE_EXPOSURE_END_DATE is not null and de.DEVICE_EXPOSURE_END_DATE > d.death_date + interval '60' day) or
482+
(de.DEVICE_EXPOSURE_START_DATE is not null and de.DEVICE_EXPOSURE_START_DATE > d.death_date + interval '60' day) or
483+
(de.DEVICE_EXPOSURE_END_DATETIME is not null and cast(de.DEVICE_EXPOSURE_END_DATETIME as date) > d.death_date + interval '60' day) or
484+
(de.DEVICE_EXPOSURE_START_DATETIME is not null and cast(de.DEVICE_EXPOSURE_START_DATETIME as date) > d.death_date + interval '60' day)
485+
)
483486
);
484487
485488
");
@@ -489,14 +492,17 @@ SELECT person_id FROM cdm.DEVICE_EXPOSURE
489492

490493
await connection.ExecuteAsync(@"
491494
492-
DELETE FROM cdm.death
493-
WHERE person_id IN (
494-
SELECT person_id FROM cdm.PROCEDURE_OCCURRENCE
495-
WHERE
496-
(PROCEDURE_DATE IS NOT NULL AND PROCEDURE_DATE > (death.death_date + INTERVAL '60' DAY)) OR
497-
(PROCEDURE_END_DATE IS NOT NULL AND PROCEDURE_END_DATE > (death.death_date + INTERVAL '60' DAY)) OR
498-
(PROCEDURE_DATETIME IS NOT NULL AND CAST(PROCEDURE_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY)) OR
499-
(PROCEDURE_END_DATETIME IS NOT NULL AND CAST(PROCEDURE_END_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
495+
delete from cdm.death d
496+
where exists (
497+
select 1
498+
from cdm.PROCEDURE_OCCURRENCE po
499+
where po.person_id = d.person_id
500+
and (
501+
(po.PROCEDURE_DATE is not null and po.PROCEDURE_DATE > d.death_date + interval '60' day) or
502+
(po.PROCEDURE_END_DATE is not null and po.PROCEDURE_END_DATE > d.death_date + interval '60' day) or
503+
(po.PROCEDURE_DATETIME is not null and cast(po.PROCEDURE_DATETIME as date) > d.death_date + interval '60' day) or
504+
(po.PROCEDURE_END_DATETIME is not null and cast(po.PROCEDURE_END_DATETIME as date) > d.death_date + interval '60' day)
505+
)
500506
);
501507
502508
");
@@ -506,14 +512,17 @@ SELECT person_id FROM cdm.PROCEDURE_OCCURRENCE
506512

507513
await connection.ExecuteAsync(@"
508514
509-
DELETE FROM cdm.death
510-
WHERE person_id IN (
511-
SELECT person_id FROM cdm.VISIT_DETAIL
512-
WHERE
513-
(VISIT_DETAIL_END_DATE IS NOT NULL AND VISIT_DETAIL_END_DATE > (death.death_date + INTERVAL '60' DAY)) OR
514-
(VISIT_DETAIL_START_DATE IS NOT NULL AND VISIT_DETAIL_START_DATE > (death.death_date + INTERVAL '60' DAY)) OR
515-
(VISIT_DETAIL_END_DATETIME IS NOT NULL AND CAST(VISIT_DETAIL_END_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY)) OR
516-
(VISIT_DETAIL_START_DATETIME IS NOT NULL AND CAST(VISIT_DETAIL_START_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
515+
delete from cdm.death d
516+
where exists (
517+
select 1
518+
from cdm.VISIT_DETAIL vd
519+
where vd.person_id = d.person_id
520+
and (
521+
(vd.VISIT_DETAIL_END_DATE is not null and vd.VISIT_DETAIL_END_DATE > d.death_date + interval '60' day) or
522+
(vd.VISIT_DETAIL_START_DATE is not null and vd.VISIT_DETAIL_START_DATE > d.death_date + interval '60' day) or
523+
(vd.VISIT_DETAIL_END_DATETIME is not null and cast(vd.VISIT_DETAIL_END_DATETIME as date) > d.death_date + interval '60' day) or
524+
(vd.VISIT_DETAIL_START_DATETIME is not null and cast(vd.VISIT_DETAIL_START_DATETIME as date) > d.death_date + interval '60' day)
525+
)
517526
);
518527
");
519528

@@ -522,42 +531,51 @@ SELECT person_id FROM cdm.VISIT_DETAIL
522531

523532
await connection.ExecuteAsync(@"
524533
525-
DELETE FROM cdm.death
526-
WHERE person_id IN (
527-
SELECT person_id FROM cdm.VISIT_OCCURRENCE
528-
WHERE
529-
(VISIT_END_DATE IS NOT NULL AND VISIT_END_DATE > (death.death_date + INTERVAL '60' DAY)) OR
530-
(VISIT_START_DATE IS NOT NULL AND VISIT_START_DATE > (death.death_date + INTERVAL '60' DAY)) OR
531-
(VISIT_END_DATETIME IS NOT NULL AND CAST(VISIT_END_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY)) OR
532-
(VISIT_START_DATETIME IS NOT NULL AND CAST(VISIT_START_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
534+
delete from cdm.death d
535+
where exists (
536+
select 1
537+
from cdm.VISIT_OCCURRENCE vo
538+
where vo.person_id = d.person_id
539+
and (
540+
(vo.VISIT_END_DATE is not null and vo.VISIT_END_DATE > d.death_date + interval '60' day) or
541+
(vo.VISIT_START_DATE is not null and vo.VISIT_START_DATE > d.death_date + interval '60' day) or
542+
(vo.VISIT_END_DATETIME is not null and cast(vo.VISIT_END_DATETIME as date) > d.death_date + interval '60' day) or
543+
(vo.VISIT_START_DATETIME is not null and cast(vo.VISIT_START_DATETIME as date) > d.death_date + interval '60' day)
544+
)
533545
);
534546
");
535547

536548
_logger.LogInformation("Correcting deaths where the patient had a DRUG_EXPOSURE beyond 60 days.");
537549

538550

539551
await connection.ExecuteAsync(@"
540-
DELETE FROM cdm.death
541-
WHERE person_id IN (
542-
SELECT person_id FROM cdm.DRUG_EXPOSURE
543-
WHERE
544-
(DRUG_EXPOSURE_END_DATE IS NOT NULL AND DRUG_EXPOSURE_END_DATE > (death.death_date + INTERVAL '60' DAY)) OR
545-
(DRUG_EXPOSURE_START_DATE IS NOT NULL AND DRUG_EXPOSURE_START_DATE > (death.death_date + INTERVAL '60' DAY)) OR
546-
(DRUG_EXPOSURE_END_DATETIME IS NOT NULL AND CAST(DRUG_EXPOSURE_END_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY)) OR
547-
(DRUG_EXPOSURE_START_DATETIME IS NOT NULL AND CAST(DRUG_EXPOSURE_START_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
552+
delete from cdm.death d
553+
where exists (
554+
select 1
555+
from cdm.DRUG_EXPOSURE de
556+
where de.person_id = d.person_id
557+
and (
558+
(de.DRUG_EXPOSURE_END_DATE is not null and de.DRUG_EXPOSURE_END_DATE > d.death_date + interval '60' day) or
559+
(de.DRUG_EXPOSURE_START_DATE is not null and de.DRUG_EXPOSURE_START_DATE > d.death_date + interval '60' day) or
560+
(de.DRUG_EXPOSURE_END_DATETIME is not null and cast(de.DRUG_EXPOSURE_END_DATETIME as date) > d.death_date + interval '60' day) or
561+
(de.DRUG_EXPOSURE_START_DATETIME is not null and cast(de.DRUG_EXPOSURE_START_DATETIME as date) > d.death_date + interval '60' day)
562+
)
548563
);");
549564

550565
_logger.LogInformation("Correcting deaths where the patient had a OBSERVATION beyond 60 days.");
551566

552567

553568
await connection.ExecuteAsync(@"
554569
555-
DELETE FROM cdm.death
556-
WHERE person_id IN (
557-
SELECT person_id FROM cdm.OBSERVATION
558-
WHERE
559-
(OBSERVATION_DATE IS NOT NULL AND OBSERVATION_DATE > (death.death_date + INTERVAL '60' DAY)) OR
560-
(OBSERVATION_DATETIME IS NOT NULL AND CAST(OBSERVATION_DATETIME AS DATE) > (death.death_date + INTERVAL '60' DAY))
570+
delete from cdm.death d
571+
where exists (
572+
select 1
573+
from cdm.OBSERVATION o
574+
where o.person_id = d.person_id
575+
and (
576+
(o.OBSERVATION_DATE is not null and o.OBSERVATION_DATE > d.death_date + interval '60' day) or
577+
(o.OBSERVATION_DATETIME is not null and cast(o.OBSERVATION_DATETIME as date) > d.death_date + interval '60' day)
578+
)
561579
);
562580
563581
");
@@ -567,12 +585,13 @@ SELECT person_id FROM cdm.OBSERVATION
567585

568586
await connection.ExecuteAsync(@"
569587
570-
DELETE FROM cdm.death
571-
WHERE person_id IN (
572-
SELECT person_id FROM cdm.CONDITION_ERA
573-
WHERE
574-
CONDITION_ERA_END_DATE IS NOT NULL
575-
AND CONDITION_ERA_END_DATE > (death.death_date + INTERVAL '60' DAY)
588+
delete from cdm.death d
589+
where exists (
590+
select 1
591+
from cdm.CONDITION_ERA ce
592+
where ce.person_id = d.person_id
593+
and ce.CONDITION_ERA_END_DATE is not null
594+
and ce.CONDITION_ERA_END_DATE > d.death_date + interval '60' day
576595
);
577596
578597
");

0 commit comments

Comments
 (0)