Merge branch 'main' into vpodcqa

Author: Shawn O'Connor

.github/workflows/dependabot.yaml .github/dependabot.yaml.github/workflows/dependabot.yaml renamed to .github/dependabot.yaml

@@ -10,7 +10,7 @@ on:
 jobs:
   update-package-dependencies:
     runs-on: ubuntu-latest
-    if: github.event.pull_request.title contains 'bump @gen3/frontend'
+    if: contains(github.event.pull_request.title, 'bump @gen3/frontend')
     steps:
       - uses: actions/checkout@v6
       - name: Set up Node

.github/workflows/update_dependencies.yaml

@@ -23,19 +23,23 @@ const TestSourcesData = {
 }
 
 const TestOverlapData = {
-  cohort_overlap: { case_control_overlap: 111 }
+  cohort_overlap: { case_control_overlap: 90 }
 }
 
 const TestStatsData1 = {
   cohort_definition_and_stats: { size: 112 }
 }
 
 const TestStatsData2 = {
-  cohort_definition_and_stats: { size: 107 }
+  cohort_definition_and_stats: { size: 87 }
 }
 
 const TestStatsData3 = {
-  cohort_definition_and_stats: { size: 105 }
+  cohort_definition_and_stats: { size: 85 }
+}
+
+const TestStatsData4 = {
+  cohort_definition_and_stats: { size: 5 }
 }
 
 interface cohort { // TODO - centralize this interface
@@ -65,6 +69,7 @@ const AttritionTableWithHooks = () => {
         datasetObservationWindow={365}
         selectedOutcomeCohort={selectedOutcomeCohort}
         outcomeObservationWindow={365}
+        removeIndividualsWithPriorOutcome={true}
         percentageOfDataToUseAsTest={25}
       />
     </SourceContextProvider>
@@ -92,6 +97,9 @@ export const AttritionTableMockedSuccess: Story = {
         http.get(CohortsEndpoint + '/:sourceId/by-cohort-definition-ids/:cohort1_definition_id/:cohort2_definition_id/by-observation-window-1st-cohort/:observationwindow/by-outcome-window-2nd-cohort/:outcomeWindow2ndCohort', async () => {
           return HttpResponse.json(TestStatsData3);
         }),
+        http.get(CohortsEndpoint + '/:sourceId/by-cohort-definition-ids/:cohort1_definition_id/:cohort2_definition_id/by-observation-window-1st-cohort/:observationwindow/and-cohort2-entry-first', async () => {
+          return HttpResponse.json(TestStatsData4);
+        }),
       ],
     },
   },

src/lib/AnalysisApps/PLP/Components/AttritionTableWrapper/AttritionTable/AttritionTable.stories.tsx

@@ -10,6 +10,7 @@ interface AttritionTableProps {
   datasetObservationWindow: number;
   selectedOutcomeCohort: cohort;
   outcomeObservationWindow: number;
+  removeIndividualsWithPriorOutcome: boolean;
   percentageOfDataToUseAsTest: number | null;
 }
 
@@ -19,7 +20,7 @@ interface cohort { // TODO - centralize this interface
   size: number;
 }
 
-type Key = 'A1' | 'B1' | 'C1' | 'D1' | 'A2' | 'B2' | 'C2' | 'D2' | 'A3' | 'B3' | 'C3' | 'D3' | 'A4' | 'B4' | 'C4'| 'D4' ;
+type Key = 'A1' | 'B1' | 'C1' | 'D1' | 'A2' | 'B2' | 'C2' | 'D2' | 'A3' | 'B3' | 'C3' | 'D3' | 'A4' | 'B4' | 'C4'| 'D4' | 'A5' | 'B5' | 'C5'| 'D5';
 type ValueMap = Record<Key, number | null>;
 type LoadingMap = Record<Key, boolean>;
 const ComputeError = 1;
@@ -29,6 +30,7 @@ const cellKeys: Key[][] = [
   ['A2', 'B2', 'C2', 'D2'],
   ['A3', 'B3', 'C3', 'D3'],
   ['A4', 'B4', 'C4', 'D4'],
+  ['A5', 'B5', 'C5', 'D5'],
 ];
 
 export const AttritionTable: React.FC<AttritionTableProps> = ({
@@ -37,14 +39,16 @@ export const AttritionTable: React.FC<AttritionTableProps> = ({
   datasetObservationWindow,
   selectedOutcomeCohort,
   outcomeObservationWindow,
+  removeIndividualsWithPriorOutcome,
   percentageOfDataToUseAsTest,
 }) => {
   const { sourceId } = useSourceContext();
-  const steps = [ 1, 2, 4, 6 ]; // the workflow step related to each description below
+  const steps = [ 1, 2, '4a', '4b', 6 ]; // the workflow step related to each description below
   const descriptions = [
     'Initial data cohort',
     `Observation window (${datasetObservationWindow} days)`,
-    `Time-at-risk (${outcomeObservationWindow} days)`,
+    `Time-at-risk (${outcomeObservationWindow} days), `,
+    `Remove prior (${removeIndividualsWithPriorOutcome ? 'yes' : 'no'})`,
     `Training set (${percentageOfDataToUseAsTest? 100-percentageOfDataToUseAsTest : '...'}%)`,
   ];
 
@@ -98,11 +102,14 @@ export const AttritionTable: React.FC<AttritionTableProps> = ({
     const responseData = await response.json();
     return responseData.cohort_definition_and_stats?.size;
   };
-  const getInObservationWindowAndOverlapWithOutcome = async () => {
+  const getInObservationWindowAndOverlapWithOutcome = async (outcomeCohortEntryFirst: boolean = false) => {
     if (! (selectedStudyPopulationCohort && selectedOutcomeCohort && datasetObservationWindow) ) {
       return null;
     }
-    const endpoint = CohortsEndpoint + `/${sourceId}/by-cohort-definition-ids/${selectedStudyPopulationCohort.cohort_definition_id}/${selectedOutcomeCohort.cohort_definition_id}/by-observation-window-1st-cohort/${datasetObservationWindow}`;
+    let endpoint = CohortsEndpoint + `/${sourceId}/by-cohort-definition-ids/${selectedStudyPopulationCohort.cohort_definition_id}/${selectedOutcomeCohort.cohort_definition_id}/by-observation-window-1st-cohort/${datasetObservationWindow}`;
+    if (outcomeCohortEntryFirst) {
+        endpoint += '/and-cohort2-entry-first'
+    }
     const response = await fetch(endpoint, {
       method: 'GET',
     });
@@ -128,25 +135,36 @@ export const AttritionTable: React.FC<AttritionTableProps> = ({
     return responseData.cohort_definition_and_stats?.size;
   };
 
+  const getNewDatasetSize = async (size: number, removeIndividualsWithPriorOutcome: boolean) => {
+    if (removeIndividualsWithPriorOutcome) {
+      return size - await getInObservationWindowAndOverlapWithOutcome(true);
+    } else {
+      return size;
+    }
+  };
+
   const [values, setValues] = useState<ValueMap>({
     A1: null, B1: null, C1: null, D1: null,
     A2: null, B2: null, C2: null, D2: null,
     A3: null, B3: null, C3: null, D3: null,
     A4: null, B4: null, C4: null, D4: null,
+    A5: null, B5: null, C5: null, D5: null,
   });
 
   const [errors] = useState<ValueMap>({
     A1: null, B1: null, C1: null, D1: null,
     A2: null, B2: null, C2: null, D2: null,
     A3: null, B3: null, C3: null, D3: null,
     A4: null, B4: null, C4: null, D4: null,
+    A5: null, B5: null, C5: null, D5: null,
   });
 
   const [loading, setLoading] = useState<LoadingMap>({
     A1: false, B1: false, C1: false, D1: false,
     A2: false, B2: false, C2: false, D2: false,
     A3: false, B3: false, C3: false, D3: false,
     A4: false, B4: false, C4: false, D4: false,
+    A5: false, B5: false, C5: false, D5: false,
   });
 
   const valueFns: ((vals: ValueMap) => Promise<number | null>)[][] = [
@@ -163,24 +181,30 @@ export const AttritionTable: React.FC<AttritionTableProps> = ({
       async (v) => (v.A2 == null || v.C2 == null ? null : (v.A2 - v.C2)), // D2
     ],
     [
-      async (v) =>  (v.C3 == null || v.D3 == null ? null : (v.C3 + v.D3)),                         // A3
+      async (v) =>  (v.A2 == null ? null : (v.A2 )),                                               // A3
       async () => null,                                                                            // B3
       async () => getInObservationWindowAndOverlapWithOutcomeAndInOutcomeWindow(),                 // C3
       async (v) =>  (v.D2 == null || v.C2 == null || v.C3 == null ? null : (v.D2 +(v.C2 - v.C3))), // D3
     ],
     [
-      async (v) => getAndSetRemaningSize(v.A3),                          // A4
-      async (v) => getTraininSetSize(v.A3),                              // B4
-      async (v) => getTraininSetSize(v.C3),                              // C4
-      async (v) => getTraininSetSize(v.D3),                              // D4
+      async (v) => (v.A3 == null ? null : getNewDatasetSize(v.A3, removeIndividualsWithPriorOutcome)), // A4
+      async (v) => null,                                                    // B4
+      async (v) =>  (v.C3 == null ? null : (v.C3 )),                        // C4
+      async (v) => (v.A4 == null || v.C4 == null ? null : (v.A4 - v.C4)),   // D4
+    ],
+    [
+      async (v) => getAndSetRemaningSize(v.A4),                          // A5
+      async (v) => getTraininSetSize(v.A4),                              // B5
+      async (v) => getTraininSetSize(v.C4),                              // C5
+      async (v) => getTraininSetSize(v.D4),                              // D5
     ],
   ];
 
   useEffect(() => {
     const compute = async (initialValues: ValueMap, recalculateAll: boolean) => {
       const result: ValueMap = { ...initialValues };
-      for (let row = 0; row < 4; row++) {
-        for (let col = 0; col < 4; col++) {
+      for (let row = 0; row < 5; row++) {
+        for (let col = 0; col < 5; col++) {
           const key = cellKeys[row][col];
           const fn = valueFns[row][col];
           // only compute if (still) null:
@@ -211,6 +235,7 @@ export const AttritionTable: React.FC<AttritionTableProps> = ({
     datasetObservationWindow,
     selectedOutcomeCohort,
     outcomeObservationWindow,
+    removeIndividualsWithPriorOutcome,
     percentageOfDataToUseAsTest,
   ]);
 

src/lib/AnalysisApps/PLP/Components/AttritionTableWrapper/AttritionTable/AttritionTable.tsx

@@ -8,6 +8,7 @@ interface AttritionTableWrapperProps {
   datasetObservationWindow: number;
   selectedOutcomeCohort: cohort;
   outcomeObservationWindow: number;
+  removeIndividualsWithPriorOutcome: boolean;
   percentageOfDataToUseAsTest: number | null;
 }
 
@@ -23,6 +24,7 @@ const AttritionTableWrapper: React.FC<AttritionTableWrapperProps> = ({
   datasetObservationWindow,
   selectedOutcomeCohort,
   outcomeObservationWindow,
+  removeIndividualsWithPriorOutcome,
   percentageOfDataToUseAsTest,
 }) => {
   const [isOpen, setIsOpen] = useState(false);
@@ -56,6 +58,7 @@ const AttritionTableWrapper: React.FC<AttritionTableWrapperProps> = ({
               datasetObservationWindow={datasetObservationWindow}
               selectedOutcomeCohort={selectedOutcomeCohort}
               outcomeObservationWindow={outcomeObservationWindow}
+              removeIndividualsWithPriorOutcome={removeIndividualsWithPriorOutcome}
               percentageOfDataToUseAsTest={percentageOfDataToUseAsTest}
               />
           </div> : null}

src/lib/AnalysisApps/PLP/Components/AttritionTableWrapper/AttritionTableWrapper.tsx

@@ -41,6 +41,7 @@ const AttritionTableWithHooks = () => {
         datasetObservationWindow={state.datasetObservationWindow}
         selectedOutcomeCohort={state.selectedOutcomeCohort}
         outcomeObservationWindow={state.outcomeObservationWindow}
+        removeIndividualsWithPriorOutcome={state.removeIndividualsWithPriorOutcome}
         selectedTeamProject={state.selectedTeamProject}
         minimumCovariateOccurrence={state.minimumCovariateOccurrence}
         percentageOfDataToUseAsTest={25}

src/lib/AnalysisApps/PLP/Components/JobSubmitModal/JobSubmitModal.stories.tsx

@@ -12,6 +12,7 @@ interface Props {
   datasetObservationWindow: number;
   selectedOutcomeCohort: cohort;
   outcomeObservationWindow: number;
+  removeIndividualsWithPriorOutcome: boolean;
   selectedTeamProject: string;
   minimumCovariateOccurrence: number;
   percentageOfDataToUseAsTest: number;
@@ -34,6 +35,7 @@ const JobSubmitModal: React.FC<Props> = ({
   datasetObservationWindow,
   selectedOutcomeCohort,
   outcomeObservationWindow,
+  removeIndividualsWithPriorOutcome,
   selectedTeamProject,
   minimumCovariateOccurrence,
   percentageOfDataToUseAsTest,
@@ -91,7 +93,7 @@ const JobSubmitModal: React.FC<Props> = ({
         min_time_at_risk: 364,  // TODO - advanced option
         include_all_outcomes:  true, // TODO - advanced option
         first_exposure_only: false, // TODO - advanced option
-        remove_subjects_with_prior_outcome: false, // TODO - advanced option
+        remove_subjects_with_prior_outcome: removeIndividualsWithPriorOutcome,
         source_id: sourceId,
         covariate_min_fraction: minimumCovariateOccurrence,
         test_fraction: percentageOfDataToUseAsTest/100,
@@ -208,6 +210,14 @@ const JobSubmitModal: React.FC<Props> = ({
                   {outcomeObservationWindow} days
                 </td>
               </tr>
+              <tr>
+                <td className="font-semibold pr-4 text-right align-top whitespace-nowrap">
+                  Remove indiv. with prior outcome:
+                </td>
+                <td className="align-top">
+                  {removeIndividualsWithPriorOutcome ? 'Yes' : 'No'}
+                </td>
+              </tr>
               <tr>
                 <td className="font-semibold pr-4 text-right align-top whitespace-nowrap">
                   Dataset size (after time window filters):

src/lib/AnalysisApps/PLP/Components/JobSubmitModal/JobSubmitModal.tsx

@@ -79,11 +79,13 @@ const PLPContainer = () => {
           <div>
             In this step, you can define an outcome period in days. This period specifies how many days to look for the outcome
             of interest to occur for each patient relative to their cohort entry date in the initial dataset.
-            This is also known as the time-at-risk window.
+            This is also known as the outcome window or time-at-risk window.
+            Here you can also choose remove any individuals that have any record of the outcome in the period <i>before</i> the outcome window.
           </div>
           <br/>
           <DefineOutcomeObservationWindow
             outcomeObservationWindow={state.outcomeObservationWindow}
+            removeIndividualsWithPriorOutcome={state.removeIndividualsWithPriorOutcome}
             dispatch={dispatch}
           />
           <br/>
@@ -157,6 +159,7 @@ const PLPContainer = () => {
               datasetObservationWindow={state.datasetObservationWindow}
               selectedOutcomeCohort={state.selectedOutcomeCohort}
               outcomeObservationWindow={state.outcomeObservationWindow}
+              removeIndividualsWithPriorOutcome={state.removeIndividualsWithPriorOutcome}
               selectedTeamProject={state.selectedTeamProject}
               minimumCovariateOccurrence={state.minimumCovariateOccurrence}
               percentageOfDataToUseAsTest={state.percentageOfDataToUseAsTest}
@@ -210,6 +213,7 @@ const PLPContainer = () => {
         datasetObservationWindow={state.datasetObservationWindow}
         selectedOutcomeCohort={state.selectedOutcomeCohort}
         outcomeObservationWindow={state.outcomeObservationWindow}
+        removeIndividualsWithPriorOutcome={state.removeIndividualsWithPriorOutcome}
         percentageOfDataToUseAsTest={state.percentageOfDataToUseAsTest}
       />
       <div data-testid="GWASApp" className="p-4">

src/lib/AnalysisApps/PLP/PLPContainer.tsx

@@ -27,6 +27,9 @@ const AddCovariates = ({
         min={0.1}
         step={0.1}
         defaultValue={0.1}
+        classNames={{
+          section: 'text-gray-500'
+        }}
         rightSection="%"
         value={minimumCovariateOccurrence * 100} // Convert decimal to percentage
         onChange={(value) => {

src/lib/AnalysisApps/PLP/Steps/AddCovariates/AddCovariates.tsx

@@ -25,6 +25,9 @@ const DefineDatasetObservationWindow = ({
       <NumberInput
         label="Dataset observation window "
         placeholder="Enter number of days"
+        classNames={{
+          section: 'text-gray-500'
+        }}
         w={400}
         min={0}
         value={datasetObservationWindow}