update

Author: vojtechhuser

DESCRIPTION

@@ -18,7 +18,7 @@ Imports:
 License: Apache License 2.0
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 6.1.1
+RoxygenNote: 7.0.2
 Suggests: knitr,
     rmarkdown
 VignetteBuilder: knitr

R/StudySpecific.R

@@ -509,6 +509,7 @@ GROUP BY o.stratum1_id, o.stratum2_id, o.total, o.min_value, o.max_value, o.avg_
 #' Execute all components of the DataQuality study (resultsDatabaseSchema is where Achilles results are)
 #' @param connectionDetails connection
 #' @param connectionDetails2 more study parameters
+#' @param runViaAchilles flag that runs legacy analysis using Achilles measure (analysis)
 
 #' @export
 dashboardLabThresholds <- function(connectionDetails,

R/TestCode.R

@@ -2,6 +2,23 @@
 
 This is an R package that has sume utilities and also supports  informatics studies that focuses on data quality (rather than a clinical question). It may also support development of other tools.
 
+
+# Support development of Data Quality Dashboard (DQD)
+
+Package has functions that support addition of more rules and knowledge base items to the Data Quality Dashboard. For example, we want to compute 3rd and 97th percentile (or other parameters) for some lab results to use those as thresholds.
+
+A site can participate on developing knowledbe base for DQD. Or it can take part on format research study (to be published). Se [extras/protocol](extras/protocol) folder for protocol. 
+
+To run the study, see DQD section in file [extras/CodeToRun.R](extras/CodeToRun.R)
+
+
+No need to read futher if you only need to run DQD study.
+
+
+
+
+# Additional info (not needed for DQD study)
+
 For the study, the protocol is available in extras folder.
 
 Forum link: http://forums.ohdsi.org/t/ohdsi-informatics-study-data-quality/1857/2
@@ -27,16 +44,6 @@ There are several  usage scenarios.
 - adding MIAD
 
 
-# Support development of Data Quality Dashboard
-
-Package has functions that support addition of more rules and knowledge base items to the Data Quality Dashboard. For example, we want to compute 3rd and 97th percentile (or other parameters) for some lab results to use those as thresholds.
-
-This is related to unit analysis done by https://github.com/vojtechhuser/ThemisConcepts
-
-A site can participate on developing knowledbe base for DQD. Or it can take part on format research study (to be published). Se [extras/protocol](extras/protocol) folder for protocol. 
-
-To run the study, see DQD section in file [extras/TestCode.R](extras/TestCode.R)
-
 
 
 # Generate MIAD (minumum information about a dataset)

README.md

@@ -46,14 +46,17 @@ load('o:/athena/concept.rda')
 
 #reading a single site data (for now) 
 f<-'d:/OneDrive - National Institutes of Health/temp/dqd/export'
+f<-'d:/OneDrive - National Institutes of Health/ohdsi/thresholds'
 
 sfiles<-c(file.path(f,'1ThresholdsA.csv'))
-sfiles<-c(file.path(f,'1ThresholdsA.csv'),file.path(f,'ThresholdsA.csv'))
+sfiles<-c(file.path(f,'test-ThresholdsA.csv'))
+sfiles<-c(file.path(f,'1ThresholdsA.csv'),file.path(f,'ThresholdsA.csv'),file.path(f,'test-ThresholdsA.csv'))
 ll<-map(sfiles,read_csv)
 ll
 
 #ll<-map(p$pid,doProperty())
-ll2<-map2(ll,sfiles,~mutate(.x,site=.y))
+#strip name from full path trick
+ll2<-map2(ll,basename(sfiles),~mutate(.x,site=.y))
 d<-bind_rows(ll2)
 
 #add terminology concepts
@@ -66,13 +69,20 @@ d2<-d %>% filter(stratum_1 != 0) %>% filter(stratum_2 != 0) %>% left_join(sconce
 names(d2)
 
 #remove columns that are not needed
-d3<-d2 %>% select(-stratum_3,-stratum_4,-stratum_5,-p25_value,-p75_value) %>% 
-  filter(count_value >=100 ) %>% arrange(stratum_1,desc(count_value) )
+# d3<-d2 %>% select(-stratum_3,-stratum_4,-stratum_5,-p25_value,-p75_value) %>% 
+#   filter(count_value >=100 ) %>% arrange(stratum_1,desc(count_value) )
+
+d3<-d2 %>% select(-stratum_3,-stratum_4,-stratum_5) %>% 
+   arrange(stratum_1,desc(count_value) )
+
 
 d3 %>% count(site)
-ba<-d3 %>% count(stratum_1,stratum_2)
+names(d3)
+ba<-d3 %>% group_by(stratum_1,stratum_2,concept_name.x,concept_name.y) %>% summarize(tcnt=sum(count_value),n=n())
 ba %>% filter(n>=2)
 
+
+
 #24 test-unit pairs  have 2 results
 
 
@@ -149,3 +159,17 @@ over %>% select(conceptName,unitConceptName,plausibleValueHigh,max_value)
 
 #5g/dL into  mg/dL  (is 5000 mg/dL)
 #in data is in fact unit/L
+
+
+
+#unitmorph
+#Protein [Mass/volume] in Serum or Plasma	7096851	4	gram per deciliter|unit|milligram per deciliter|gram per liter
+#	gram per deciliter|   |milligram per deciliter |   gram per liter
+names(d3)
+bb<-d3 %>% filter(site=='ThresholdsA.csv') %>% group_by(stratum_1,concept_name.x) %>% 
+  summarize(tcnt=sum(count_value)
+            ,n=n(),units=paste(concept_name.y,collapse = '|')
+            ,cnts=paste(count_value ,collapse = '|')
+            ,unitcids=paste(stratum_2,collapse = '|')
+  )
+bb %>% write_csv('local/morphA.csv')

extras/CentralProcessing.R

@@ -1,61 +1,42 @@
-#' @file TestCode.R
-
-
-library(DataQuality)
-
-workFolder <- 'c:/temp'
-
-#get connection details
-source('c:/r/conn.R')  #
-
-#database parameters
-cdmDatabaseSchema <-'ccae'
-resultsDatabaseSchema <-'ccae' #at most sites this likely will not be the same as cdmDatabaseSchema
-
-workFolder <- 'c:/temp'   #this folder must exist (use forward slashes)
-
-
-executeDQ(connectionDetails = connectionDetails,cdmDatabaseSchema = cdmDatabaseSchema,workFolder = workFolder)
-cd2<-.createConnectionDetails2(cdmDatabaseSchema = cdmDatabaseSchema)
-
-
-#DatabaseConnector::createConnectionDetails()
-#packageResults(connectionDetails,cdmDatabaseSchema,workFolder)
-
-submitResults(exportFolder =file.path(workFolder,'export'),
-              studyBucketName = 'ohdsi-study-dataquality',
-              key=studyKey,
-              secret =studySecret
-              )
+#' @file CodeToRun.R
 
 
 #DQD section-----------------------------------------------
 #----------------------------------------------------------
 #----------------------------------------------------------
+
+
+#---LOAD THE R PACKAGE
 #uncomment the line below to install the latest version of the code and package
 #devtools::install_github("vojtechhuser/DataQuality")
-#testing it on Eunomia
+
+
+
+#load the package 
 library(DataQuality)
 
 
+#---SPECIFY YOUR LOCAL PARAMTERS BELOW-----
 #replace with your local parameters using the same variable names
 #note: it will run some achilles measures (so it may overwrite your current achilles results table)
-library(Eunomia)
 connectionDetails<-Eunomia::getEunomiaConnectionDetails() 
 cdmDatabaseSchema <-'main'
 resultsDatabaseSchema <-'main' #at most sites this likely will not be the same as cdmDatabaseSchema
 workFolder <- 'c:/temp/dqd'   #this folder must exist (use forward slashes)
 
-#just a helper construct to package all study details into one object (similar as done with database details)
+#just a helper construct to package all study details into one object (similar as done with database connection details)
 connectionDetails2<-DataQuality:::.createConnectionDetails2(cdmDatabaseSchema = cdmDatabaseSchema
                                               ,resultsDatabaseSchema = resultsDatabaseSchema
                                               ,workFolder = workFolder
                                               )
 
+
+#----EXECUTE THE DQD THRESHOLD STUDY
 #execute the core of the DQD helper analysis, outptu will be writen as CSV file into an export folder
 
 DataQuality::dashboardLabThresholds(connectionDetails = connectionDetails,connectionDetails2 = connectionDetails2)
 
-#results will be in workfolder, subfolder export (will be zipped once we have more output)
+#results are in workfolder, subfolder export, inspect it and 
+#email the CSV file Thresholds to the study PI