Merge pull request #159 from X-DataInitiative/CNAM-344-simplify-study-output

dsun0720 · web-flow · commit f1022032d026 · 2018-08-23T11:11:09.000+02:00
Cnam 344 simplify study output
diff --git a/src/main/resources/config/fall/paths/cmap.conf b/src/main/resources/config/fall/paths/cmap.conf
@@ -1,8 +1,12 @@
 env_name = "cmap"
 
 input = {
-
+  dcir = "/shared/Observapur/staging/Flattening/flat_table/DCIR"
+  mco = "/shared/Observapur/staging/Flattening/flat_table/MCO"
+  ir_ben = "/shared/Observapur/staging/Flattening/single_table/IR_BEN_R"
+  ir_imb = "/shared/Observapur/staging/Flattening/single_table/IR_IMB_R"
+  ir_pha = "/shared/Observapur/staging/Flattening/single_table/IR_PHA_R_MOL"
 }
 output = {
-
+  root = "/shared/fall"
 }
diff --git a/src/main/resources/config/fall/paths/cnam.conf b/src/main/resources/config/fall/paths/cnam.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "/shared/FALL/CNAM278/featuring_results/filtered_data"
-  patients = "/shared/FALL/CNAM278/featuring_results/filtered_data/patients"
-  flat_events = "/shared/FALL/CNAM278/featuring_results/filtered_data/flat_events"
-  cox_features = "/shared/FALL/CNAM278/featuring_results/features/cox_features"
-  ltsccs_features = "/shared/FALL/CNAM278/featuring_results/features/ltsccs_features"
-  mlpp_features = "/shared/FALL/CNAM278/featuring_results/features/mlpp_features"
-  outcomes = "/shared/FALL/CNAM278/featuring_results/outcomes/fractures"
-  exposures = "/shared/FALL/CNAM278/featuring_results/exposures"
 }
diff --git a/src/main/resources/config/fall/paths/test.conf b/src/main/resources/config/fall/paths/test.conf
@@ -10,11 +10,4 @@ input = {
 
 output = {
   root = "target/test/output"
-  patients = "target/test/output/patients"
-  flat_events = "target/test/output/flat_events"
-  cox_features = "target/test/output/cox_features"
-  ltsccs_features = "target/test/output/ltsccs_features"
-  mlpp_features = "target/test/output/mlpp_features"
-  outcomes = "target/test/output/outcomes/fractures"
-  exposures = "target/test/output/exposures"
 }
diff --git a/src/main/resources/config/pioglitazone/paths/cmap.conf b/src/main/resources/config/pioglitazone/paths/cmap.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "/shared/PIO/filtered_data"
-  patients = "/shared/PIO/filtered_data/patients"
-  flat_events = "/shared/PIO/filtered_data/flat_events"
-  cox_features = "/shared/PIO/features/cox_features"
-  ltsccs_features = "/shared/PIO/features/ltsccs_features"
-  mlpp_features = "/shared/PIO/featues/mlpp_features"
-  outcomes = "/shared/PIO/outcomes/cancer"
-  exposures = "/shared/PIO/exposures"
 }
diff --git a/src/main/resources/config/pioglitazone/paths/cnam.conf b/src/main/resources/config/pioglitazone/paths/cnam.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "/shared/PIO/featuring_results/filtered_data"
-  patients = "/shared/PIO/featuring_results/filtered_data/patients"
-  flat_events = "/shared/PIO/featuring_results/filtered_data/flat_events"
-  cox_features = "/shared/PIO/featuring_results/features/cox_features"
-  ltsccs_features = "/shared/PIO/featuring_results/features/ltsccs_features"
-  mlpp_features = "/shared/PIO/featuring_results/features/mlpp_features"
-  outcomes = "/shared/PIO/featuring_results/outcomes/cancer"
-  exposures = "/shared/PIO/featuring_results/exposures"
 }
diff --git a/src/main/resources/config/pioglitazone/paths/test.conf b/src/main/resources/config/pioglitazone/paths/test.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "target/test/output"
-  patients = "target/test/output/patients"
-  flat_events = "target/test/output/flat_events"
-  cox_features = "target/test/output/cox_features"
-  ltsccs_features = "target/test/output/ltsccs_features"
-  mlpp_features = "target/test/output/mlpp_features"
-  outcomes = "target/test/output/outcomes/cancer"
-  exposures = "target/test/output/exposures"
 }
diff --git a/src/main/resources/config/rosiglitazone/paths/cmap.conf b/src/main/resources/config/rosiglitazone/paths/cmap.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "/shared/ROSI/"
-  patients = "/shared/ROSI/filtered_data/patients"
-  flat_events = "/shared/ROSI/filtered_data/flat_events"
-  cox_features = "/shared/ROSI/features/cox_features"
-  ltsccs_features = "/shared/ROSI/features/ltsccs_features"
-  mlpp_features = "/shared/ROSI/featues/mlpp_features"
-  outcomes = "/shared/ROSI/outcomes/heart_problems"
-  exposures = "/shared/ROSI/exposures"
 }
diff --git a/src/main/resources/config/rosiglitazone/paths/cnam.conf b/src/main/resources/config/rosiglitazone/paths/cnam.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "/shared/ROSI/featuring_results"
-  patients = "/shared/ROSI/featuring_results/filtered_data/patients"
-  flat_events = "/shared/ROSI/featuring_results/filtered_data/flat_events"
-  cox_features = "/shared/ROSI/featuring_results/features/cox_features"
-  ltsccs_features = "/shared/ROSI/featuring_results/features/ltsccs_features"
-  mlpp_features = "/shared/ROSI/featuring_results/features/mlpp_features"
-  outcomes = "/shared/ROSI/featuring_results/outcomes/heart_problems"
-  exposures = "/shared/ROSI/featuring_results/exposures"
 }
diff --git a/src/main/resources/config/rosiglitazone/paths/test.conf b/src/main/resources/config/rosiglitazone/paths/test.conf
@@ -10,11 +10,4 @@ input = {
 }
 output = {
   root = "target/test/output"
-  patients = "target/test/output/patients"
-  flat_events = "target/test/output/flat_events"
-  cox_features = "target/test/output/cox_features"
-  ltsccs_features = "target/test/output/ltsccs_features"
-  mlpp_features = "target/test/output/mlpp_features"
-  outcomes = "target/test/output/outcomes/heart_problems"
-  exposures = "target/test/output/exposures"
 }
diff --git a/src/main/scala/fr/polytechnique/cmap/cnam/etl/config/study/StudyConfig.scala b/src/main/scala/fr/polytechnique/cmap/cnam/etl/config/study/StudyConfig.scala
@@ -3,6 +3,7 @@ package fr.polytechnique.cmap.cnam.etl.config.study
 import fr.polytechnique.cmap.cnam.etl.config.Config
 
 object StudyConfig {
+
   case class InputPaths(
     dcir: Option[String] = None,
     mco: Option[String] = None,
@@ -14,15 +15,9 @@ object StudyConfig {
     irPha: Option[String] = None,
     dosages: Option[String] = None)
 
-  case class  OutputPaths(
-    root: String,
-    patients: String,
-    flatEvents: String,
-    coxFeatures: String,
-    ltsccsFeatures: String,
-    mlppFeatures: String,
-    outcomes: String,
-    exposures: String)
+  case class OutputPaths(
+    root: String)
+
 }
 
 trait StudyConfig extends Config
diff --git a/src/main/scala/fr/polytechnique/cmap/cnam/study/fall/FallMain.scala b/src/main/scala/fr/polytechnique/cmap/cnam/study/fall/FallMain.scala
@@ -27,6 +27,7 @@ object FallMain extends Main with FractureCodes {
 
   override def run(sqlContext: SQLContext, argsMap: Map[String, String]): Option[Dataset[_]] = {
 
+    val format = new java.text.SimpleDateFormat("yyyy_MM_dd_HH_mm_ss")
     val startTimestamp = new java.util.Date()
     val fallConfig = FallConfig.load(argsMap("conf"), argsMap("env"))
 
@@ -50,7 +51,7 @@ object FallMain extends Main with FractureCodes {
     // Extract Patients
     val patients = new Patients(PatientsConfig(fallConfig.base.studyStart)).extract(sources).cache()
     operationsMetadata += {
-      OperationReporter.report("extract_patients", List("DCIR", "MCO", "IR_BEN_R"), OperationTypes.Patients, patients.toDF, Path(fallConfig.output.patients))
+      OperationReporter.report("extract_patients", List("DCIR", "MCO", "IR_BEN_R"), OperationTypes.Patients, patients.toDF, Path(fallConfig.output.root))
     }
 
     // Filter Patients
@@ -68,7 +69,7 @@ object FallMain extends Main with FractureCodes {
       new ExposuresTransformer(definition).transform(patientsWithFollowUp, drugPurchases)
     }
     operationsMetadata += {
-      OperationReporter.report("exposures", List("drug_purchases"), OperationTypes.Exposures, exposures.toDF, Path(fallConfig.output.exposures))
+      OperationReporter.report("exposures", List("drug_purchases"), OperationTypes.Exposures, exposures.toDF, Path(fallConfig.output.root))
     }
     drugPurchases.unpersist()
 
@@ -96,7 +97,7 @@ object FallMain extends Main with FractureCodes {
     // fractures from All sources
     val fractures: Dataset[Event[Outcome]] = new FracturesTransformer(fallConfig).transform(liberalActs, acts, diagnoses)
     operationsMetadata += {
-      OperationReporter.report("fractures", List("acts"), OperationTypes.Outcomes, fractures.toDF, Path(fallConfig.output.outcomes))
+      OperationReporter.report("fractures", List("acts"), OperationTypes.Outcomes, fractures.toDF, Path(fallConfig.output.root))
     }
     diagnoses.unpersist()
     liberalActs.unpersist()
@@ -106,7 +107,7 @@ object FallMain extends Main with FractureCodes {
     val metadata = MainMetadata(this.getClass.getName, startTimestamp, new java.util.Date(), operationsMetadata.toList)
     val metadataJson: String = metadata.toJsonString()
 
-    new PrintWriter("metadata.json") {
+    new PrintWriter("metadata_fall_" + format.format(startTimestamp) + ".json") {
       write(metadataJson)
       close()
     }
diff --git a/src/main/scala/fr/polytechnique/cmap/cnam/study/pioglitazone/PioglitazoneMain.scala b/src/main/scala/fr/polytechnique/cmap/cnam/study/pioglitazone/PioglitazoneMain.scala
@@ -36,6 +36,7 @@ object PioglitazoneMain extends Main {
     */
   def run(sqlContext: SQLContext, argsMap: Map[String, String] = Map()): Option[Dataset[_]] = {
 
+    val format = new java.text.SimpleDateFormat("yyyy_MM_dd_HH_mm_ss")
     val startTimestamp = new java.util.Date()
     val operationsMetadata = mutable.Buffer[OperationMetadata]()
 
@@ -225,7 +226,7 @@ object PioglitazoneMain extends Main {
     val metadata = MainMetadata(this.getClass.getName, startTimestamp, new java.util.Date(), operationsMetadata.toList)
     val metadataJson: String = metadata.toJsonString()
 
-    new PrintWriter("metadata_pioglitazone_" + startTimestamp.toString + ".json") {
+    new PrintWriter("metadata_pioglitazone_" + format.format(startTimestamp) + ".json") {
       write(metadataJson)
       close()
     }
diff --git a/src/main/scala/fr/polytechnique/cmap/cnam/study/rosiglitazone/RosiglitazoneMain.scala b/src/main/scala/fr/polytechnique/cmap/cnam/study/rosiglitazone/RosiglitazoneMain.scala
@@ -29,6 +29,7 @@ object RosiglitazoneMain extends Main {
 
   def run(sqlContext: SQLContext, argsMap: Map[String, String] = Map()): Option[Dataset[_]] = {
 
+    val format = new java.text.SimpleDateFormat("yyyy_MM_dd_HH_mm_ss")
     val startTimestamp = new java.util.Date()
     import sqlContext.implicits._
     import PatientFilters._
@@ -210,7 +211,7 @@ object RosiglitazoneMain extends Main {
     val metadata = MainMetadata(this.getClass.getName, startTimestamp, new java.util.Date(), operationsMetadata.toList)
     val metadataJson: String = metadata.toJsonString()
 
-    new PrintWriter("metadata_rosiglitazone_" + startTimestamp.toString + ".json") {
+    new PrintWriter("metadata_rosiglitazone_" + format.format(startTimestamp) + ".json") {
       write(metadataJson)
       close()
     }
diff --git a/src/test/scala/fr/polytechnique/cmap/cnam/study/fall/config/FallConfigSuite.scala b/src/test/scala/fr/polytechnique/cmap/cnam/study/fall/config/FallConfigSuite.scala
@@ -19,14 +19,7 @@ class FallConfigSuite extends FlatSpec{
   )
 
   val outputPaths = OutputPaths(
-    root = "target/test/output",
-    patients = "target/test/output/patients",
-    flatEvents = "target/test/output/flat_events",
-    coxFeatures = "target/test/output/cox_features",
-    ltsccsFeatures = "target/test/output/ltsccs_features",
-    mlppFeatures = "target/test/output/mlpp_features",
-    outcomes = "target/test/output/outcomes/fractures",
-    exposures = "target/test/output/exposures"
+    root = "target/test/output"
   )
   "load" should "load default config file" in {
     //Given
diff --git a/src/test/scala/fr/polytechnique/cmap/cnam/study/pioglitazone/PioglitazoneConfigSuite.scala b/src/test/scala/fr/polytechnique/cmap/cnam/study/pioglitazone/PioglitazoneConfigSuite.scala
@@ -19,14 +19,7 @@ class PioglitazoneConfigSuite extends FlatSpec {
   )
 
   val outputPaths = OutputPaths(
-    root = "target/test/output",
-    patients = "target/test/output/patients",
-    flatEvents = "target/test/output/flat_events",
-    coxFeatures = "target/test/output/cox_features",
-    ltsccsFeatures = "target/test/output/ltsccs_features",
-    mlppFeatures = "target/test/output/mlpp_features",
-    outcomes = "target/test/output/outcomes/cancer",
-    exposures = "target/test/output/exposures"
+    root = "target/test/output"
   )
 
   "load" should "correctly load the default configuration" in {
diff --git a/src/test/scala/fr/polytechnique/cmap/cnam/study/rosiglitazone/RosiglitazoneConfigSuite.scala b/src/test/scala/fr/polytechnique/cmap/cnam/study/rosiglitazone/RosiglitazoneConfigSuite.scala
@@ -19,14 +19,7 @@ class RosiglitazoneConfigSuite extends FlatSpec {
   )
 
   val outputPaths = OutputPaths(
-    root = "target/test/output",
-    patients = "target/test/output/patients",
-    flatEvents = "target/test/output/flat_events",
-    coxFeatures = "target/test/output/cox_features",
-    ltsccsFeatures = "target/test/output/ltsccs_features",
-    mlppFeatures = "target/test/output/mlpp_features",
-    outcomes = "target/test/output/outcomes/heart_problems",
-    exposures = "target/test/output/exposures"
+    root = "target/test/output"
   )
 
   "load" should "correctly load the default configuration" in {

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,12 @@`
`1`	`1`	`env_name = "cmap"`
`2`	`2`
`3`	`3`	`input = {`
`4`		`-`
	`4`	`+ dcir = "/shared/Observapur/staging/Flattening/flat_table/DCIR"`
	`5`	`+ mco = "/shared/Observapur/staging/Flattening/flat_table/MCO"`
	`6`	`+ ir_ben = "/shared/Observapur/staging/Flattening/single_table/IR_BEN_R"`
	`7`	`+ ir_imb = "/shared/Observapur/staging/Flattening/single_table/IR_IMB_R"`
	`8`	`+ ir_pha = "/shared/Observapur/staging/Flattening/single_table/IR_PHA_R_MOL"`
`5`	`9`	`}`
`6`	`10`	`output = {`
`7`		`-`
	`11`	`+ root = "/shared/fall"`
`8`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,11 +10,4 @@ input = {`
`10`	`10`	`}`
`11`	`11`	`output = {`
`12`	`12`	`root = "/shared/FALL/CNAM278/featuring_results/filtered_data"`
`13`		`- patients = "/shared/FALL/CNAM278/featuring_results/filtered_data/patients"`
`14`		`- flat_events = "/shared/FALL/CNAM278/featuring_results/filtered_data/flat_events"`
`15`		`- cox_features = "/shared/FALL/CNAM278/featuring_results/features/cox_features"`
`16`		`- ltsccs_features = "/shared/FALL/CNAM278/featuring_results/features/ltsccs_features"`
`17`		`- mlpp_features = "/shared/FALL/CNAM278/featuring_results/features/mlpp_features"`
`18`		`- outcomes = "/shared/FALL/CNAM278/featuring_results/outcomes/fractures"`
`19`		`- exposures = "/shared/FALL/CNAM278/featuring_results/exposures"`
`20`	`13`	`}`