Skip to content

Commit 28f199b

Browse files
GahhhGahhh
committed
change upload process
1 parent 8bfadf9 commit 28f199b

File tree

6 files changed

+202
-78
lines changed

6 files changed

+202
-78
lines changed

‎scripts/db/update_db_schema_v1.sql

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
ALTER TABLE prescription
2+
RENAME COLUMN rt_plan_path TO rt_plan_pres;
3+
4+
ALTER TABLE prescription
5+
RENAME COLUMN rt_ct_path TO rt_ct_pres;
6+
7+
ALTER TABLE prescription
8+
RENAME COLUMN rt_structure_path TO rt_structure_pres;
9+
10+
ALTER TABLE prescription
11+
ALTER COLUMN rt_dose_path TO rt_dose_pres;
12+
13+
ALTER TABLE prescription
14+
ALTER COLUMN rt_mri_path TO rt_mri_pres;
15+
16+
ALTER TABLE prescription
17+
ALTER COLUMN RT_DVH_original_path TO planned_dvh_pres;
18+
19+
ALTER TABLE prescription
20+
ADD COLUMN centroid_pres VARCHAR;
21+
22+
ALTER TABLE prescription
23+
ADD COLUMN planned_dicom_pres VARCHAR;
24+
25+
ALTER TABLE images
26+
ADD COLUMN rt_ct_fraction VARCHAR;
27+
28+
ALTER TABLE images
29+
ADD COLUMN rt_dose_fraction VARCHAR;
30+
31+
ALTER TABLE images
32+
ADD COLUMN rt_plan_fraction VARCHAR;
33+
34+
ALTER TABLE images
35+
ADD COLUMN rt_structure_fraction VARCHAR;
36+
37+
ALTER TABLE images
38+
ADD COLUMN centroid_fraction VARCHAR;
39+
40+
ALTER TABLE images
41+
ADD COLUMN kim_threshold VARCHAR;
42+
43+
ALTER TABLE images
44+
ADD COLUMN planned_dvh_fraction VARCHAR;
45+
46+
ALTER TABLE images
47+
ADD COLUMN planned_dicom_fraction VARCHAR;
48+
49+
ALTER TABLE images
50+
ADD COLUMN rpm_path VARCHAR;
51+
52+
-- ALTER TABLE images
53+
-- ADD COLUMN fluoro_images_path VARCHAR;
54+
55+
-- ALTER TABLE images
56+
-- ADD COLUMN cbct_images_path VARCHAR;
57+
58+
-- ALTER TABLE images
59+
-- ADD COLUMN timepoint_ct_path VARCHAR;
60+
61+
-- ALTER TABLE images
62+
-- ADD COLUMN pet_iamges_path VARCHAR;

‎src/admin_console/CustomTypes.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,8 @@ class DBUpdateResult(NamedTuple):
2020
success: bool
2121
rowsUpdated: int
2222
message: str
23+
24+
class DBFindResult(NamedTuple):
25+
success: bool
26+
result : list
27+
message: str

‎src/admin_console/DataImporter.py

Lines changed: 69 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -118,59 +118,84 @@ def insertMetadataIntoDatabase(self, dbProgressCallback:Callable[[str], None]=No
118118
if not self.contentCopied and self.metadata["upload_type"] == "files":
119119
return False, "Please ensure that the uploaded files are copied first before inserting in DB"
120120

121-
try:
122-
with open("filetype_db_mapping.json", 'r') as filetypeMappingFile:
123-
filetypeMapping = json.load(filetypeMappingFile)
124-
except FileNotFoundError as err:
125-
return False, "The filetype mapping JSON cannot be loaded"
121+
# try:
122+
# with open("filetype_db_mapping.json", 'r') as filetypeMappingFile:
123+
# filetypeMapping = json.load(filetypeMappingFile)
124+
# except FileNotFoundError as err:
125+
# return False, "The filetype mapping JSON cannot be loaded"
126126

127127
for UploadDetails in self.metadata["uploaded_files"]:
128-
if UploadDetails["file_type"] in filetypeMapping["mapping"].keys():
129-
paths = []
130-
multiValues = filetypeMapping["mapping"][UploadDetails["file_type"]]["multivalues"]
131-
granularity = filetypeMapping["mapping"][UploadDetails["file_type"]]["granularity"]
132-
if multiValues:
133-
seperator = filetypeMapping["mapping"][UploadDetails["file_type"]]["delimiter"]
134-
for filePath in UploadDetails["Files"]:
135-
parentPath, sep, fileName = filePath.rpartition('/')
136-
if granularity == "folder":
137-
if parentPath not in paths:
138-
paths.append(parentPath)
139-
else:
140-
paths.append(filePath)
141-
if not multiValues:
142-
break
143-
fieldContent = ""
144-
if multiValues and len(paths) > 1:
145-
for path in paths:
146-
if fieldContent != "":
147-
fieldContent += seperator
148-
fieldContent += path[len(self.currentContextId):]
149-
else:
150-
fieldContent += paths[0][len(self.currentContextId):]
151-
tableName = filetypeMapping["mapping"][UploadDetails["file_type"]]["table"]
152-
fieldName = filetypeMapping["mapping"][UploadDetails["file_type"]]["field"]
153-
insertStmt = f"UPDATE {tableName} SET {fieldName} = \'{fieldContent}\' "
154-
if tableName == "prescription":
155-
insertStmt += "FROM patient " \
128+
findTableNameSql = f"SELECT table_name from information_schema.columns where column_name=\'{UploadDetails['file_type']}\'"
129+
result = self.dbAdapter.executeFindOnImageDB(findTableNameSql)
130+
if not result.success:
131+
return result.success, result.message
132+
tableName = result.result[0][0]
133+
insertStmt = ""
134+
if tableName == "prescription":
135+
insertStmt = f"UPDATE {tableName} SET {UploadDetails['file_type']} = \'{UploadDetails['folder_path'][0]}\' "
136+
insertStmt += "FROM patient " \
156137
"WHERE patient.id=prescription.patient_id " \
157138
+ f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
158139
+ f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
159140
+ f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
160-
elif tableName == "images":
161-
insertStmt += "FROM patient, prescription, fraction " \
162-
+ "WHERE patient.id=prescription.patient_id " \
163-
+ "AND prescription.prescription_id=fraction.prescription_id " \
164-
+ "AND images.fraction_id=fraction.fraction_id " \
165-
+ f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
166-
+ f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
167-
+ f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
168141
result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
169142
if not result.success:
170143
return result.success, result.message
171-
elif dbProgressCallback:
172-
dbProgressCallback(f"updated {tableName}.{fieldName} = {fieldContent}")
173-
self.markPacketAsImported()
144+
else:
145+
for fraction in UploadDetails["fraction"]:
146+
fractionDetail = self.dbAdapter.getFractionIdAndName(self.metadata["patient_trial_id"], fraction)
147+
if fractionDetail:
148+
fractionId = fractionDetail[0][0]
149+
insertStmt = f"UPDATE {tableName} SET {UploadDetails['file_type']} = \'{UploadDetails['folder_path'][fraction]}\' WHERE fraction_id = \'{fractionId}\'"
150+
result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
151+
if not result.success:
152+
return result.success, result.message
153+
# if UploadDetails["file_type"] in filetypeMapping["mapping"].keys():
154+
# paths = []
155+
# multiValues = filetypeMapping["mapping"][UploadDetails["file_type"]]["multivalues"]
156+
# granularity = filetypeMapping["mapping"][UploadDetails["file_type"]]["granularity"]
157+
# if multiValues:
158+
# seperator = filetypeMapping["mapping"][UploadDetails["file_type"]]["delimiter"]
159+
# for filePath in UploadDetails["Files"]:
160+
# parentPath, sep, fileName = filePath.rpartition('/')
161+
# if granularity == "folder":
162+
# if parentPath not in paths:
163+
# paths.append(parentPath)
164+
# else:
165+
# paths.append(filePath)
166+
# if not multiValues:
167+
# break
168+
# fieldContent = ""
169+
# if multiValues and len(paths) > 1:
170+
# for path in paths:
171+
# if fieldContent != "":
172+
# fieldContent += seperator
173+
# fieldContent += path[len(self.currentContextId):]
174+
# else:
175+
# fieldContent += paths[0][len(self.currentContextId):]
176+
# tableName = filetypeMapping["mapping"][UploadDetails["file_type"]]["table"]
177+
# fieldName = filetypeMapping["mapping"][UploadDetails["file_type"]]["field"]
178+
# insertStmt = f"UPDATE {tableName} SET {fieldName} = \'{fieldContent}\' "
179+
# if tableName == "prescription":
180+
# insertStmt += "FROM patient " \
181+
# "WHERE patient.id=prescription.patient_id " \
182+
# + f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
183+
# + f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
184+
# + f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
185+
# elif tableName == "images":
186+
# insertStmt += "FROM patient, prescription, fraction " \
187+
# + "WHERE patient.id=prescription.patient_id " \
188+
# + "AND prescription.prescription_id=fraction.prescription_id " \
189+
# + "AND images.fraction_id=fraction.fraction_id " \
190+
# + f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
191+
# + f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
192+
# + f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
193+
# result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
194+
# if not result.success:
195+
# return result.success, result.message
196+
# elif dbProgressCallback:
197+
# dbProgressCallback(f"updated {tableName}.{fieldName} = {fieldContent}")
198+
# self.markPacketAsImported()
174199
return True, "Success"
175200

176201
def rejectUploadPacket(self):

‎src/admin_console/DatabaseAdapter.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import psycopg2 as pg
33
import config
44
from typing import Dict, List, Tuple
5-
from CustomTypes import SiteDetails, TrialDetails, DBUpdateResult
5+
from CustomTypes import SiteDetails, TrialDetails, DBUpdateResult, DBFindResult
66
import sys
77

88

@@ -196,6 +196,20 @@ def executeUpdateOnImageDB(self, stmt:str) -> DBUpdateResult:
196196
return DBUpdateResult(success=False, rowsUpdated=0, message=str(error))
197197
return DBUpdateResult(success=True, rowsUpdated=rowsUpdated, message="Success")
198198

199+
def executeFindOnImageDB(self, stmt:str) -> List[Dict]:
200+
print("Executing SELECT Statement:", stmt)
201+
result = []
202+
try:
203+
conn = self.getImageDBConnection()
204+
cur = conn.cursor()
205+
cur.execute(stmt)
206+
result = cur.fetchall()
207+
cur.close()
208+
except (Exception, pg.DatabaseError) as error:
209+
print(error, file=sys.stderr)
210+
return DBFindResult(success=False, result=[], message=str(error))
211+
return DBFindResult(success=True, result=result, message="Success")
212+
199213
def getFractionIdAndDate(self, patientTrialId:str, fractionNumber:int) -> tuple:
200214
strQuery = "SELECT fraction_id, fraction_date FROM fraction, patient, prescription " \
201215
+ "WHERE patient.patient_trial_id = '" + patientTrialId + "' " \

‎src/admin_console/application.py

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -98,32 +98,31 @@ def importUploadPacket(upload_id):
9898
if config.APP_DEBUG_MODE:
9999
print("Copied files into storage", result[1])
100100
fileInfo = di.getUploadFileInfo()
101-
if fileInfo['clinical_trial'] == "CHIRP":
102-
# Only design for CHIRP data import
103-
result = di.insertCHIRPDataIntoDatabase()
104-
elif fileInfo['file_type'] == "fraction_folder":
105-
result = di.insertFractionDataIntoDatabase()
106-
result2 = di.insertImagePathIntoDatabase()
107-
if config.APP_DEBUG_MODE:
108-
print("Inserted fraction data into database", result[1])
109-
print("Inserted image path into database", result2[1])
110-
elif fileInfo['file_type'] == "image_folder":
111-
result = di.checkAndInsertFractionDataIntoDatabase()
112-
result2 = di.insertPatientLevelImagePathIntoDatabase()
113-
elif fileInfo['file_type'] == "trajectory_log_folder":
114-
result = di.insertTrajectoryLogIntoDatabase()
115-
elif fileInfo['file_type'] == "DVH_folder" or fileInfo['file_type'] == "DICOM_folder":
116-
result = di.insertDoseReconstrcutionFileIntoDatabase()
117-
elif fileInfo['file_type'] == "triangulation_folder" or fileInfo['file_type'] == "kim_logs":
118-
result = di.checkAndInsertFractionDataIntoDatabase()
119-
result2 = di.insertFractionFilePathIntoDatabase()
120-
if config.APP_DEBUG_MODE:
121-
print("Inserted fraction data into database", result[1])
122-
print("Inserted image path into database", result2[1])
123-
else:
124-
result = di.insertMetadataIntoDatabase()
125-
if config.APP_DEBUG_MODE:
126-
print("Inserted metadata into database", result[1])
101+
# if fileInfo['clinical_trial'] == "CHIRP":
102+
# # Only design for CHIRP data import
103+
# result = di.insertCHIRPDataIntoDatabase()
104+
# elif fileInfo['file_type'] == "fraction_folder":
105+
# result = di.insertFractionDataIntoDatabase()
106+
# result2 = di.insertImagePathIntoDatabase()
107+
# if config.APP_DEBUG_MODE:
108+
# print("Inserted fraction data into database", result[1])
109+
# print("Inserted image path into database", result2[1])
110+
# elif fileInfo['file_type'] == "image_folder":
111+
# result = di.checkAndInsertFractionDataIntoDatabase()
112+
# result2 = di.insertPatientLevelImagePathIntoDatabase()
113+
# elif fileInfo['file_type'] == "trajectory_log_folder":
114+
# result = di.insertTrajectoryLogIntoDatabase()
115+
# elif fileInfo['file_type'] == "DVH_folder" or fileInfo['file_type'] == "DICOM_folder":
116+
# result = di.insertDoseReconstrcutionFileIntoDatabase()
117+
# elif fileInfo['file_type'] == "triangulation_folder" or fileInfo['file_type'] == "kim_logs":
118+
# result = di.checkAndInsertFractionDataIntoDatabase()
119+
# result2 = di.insertFractionFilePathIntoDatabase()
120+
# if config.APP_DEBUG_MODE:
121+
# print("Inserted fraction data into database", result[1])
122+
# print("Inserted image path into database", result2[1])
123+
result = di.insertMetadataIntoDatabase()
124+
if config.APP_DEBUG_MODE:
125+
print("Inserted metadata into database", result[1])
127126

128127
if not result[0]:
129128
print("Error importing data:", result[1])

‎src/data_service/ContentManager.py

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -698,7 +698,6 @@ def acceptAndSaveFile(self, req:request):
698698
# with open(fileTypeToPathMappingPath, "r") as pathMappingFile:
699699
# fileTypeToPathMapping = json.load(pathMappingFile)
700700
try:
701-
702701
trialDetails = getTrialStructure(metadata["clinical_trial"])
703702
filePath = trialDetails[metadata["level"]][metadata["file_type"]]['path']
704703
except KeyError as e:
@@ -711,7 +710,21 @@ def acceptAndSaveFile(self, req:request):
711710
filesSaved = []
712711
for fileFieldName in req.files.keys():
713712
uploadedFile = req.files[fileFieldName]
713+
formatedPath = ""
714+
fractionName = ""
715+
fractionNumber = ""
714716
filename = secure_filename(uploadedFile.filename)
717+
if metadata['level'] == "fraction":
718+
formatedPath = os.path.basename(req.form["file_path"]).replace("\\", "/").replace(filename, "")
719+
fractionNumber = re.search(r'(?i)fx(\d+)', formatedPath).group(1)
720+
fractionName = ""
721+
if formatedPath.count("/") == 3:
722+
fractionName = re.search(r'\/([^\/]+)\/$', formatedPath).group(1)
723+
if fractionName=="":
724+
fractionName = fractionNumber
725+
else:
726+
formatedPath = re.sub(r'^([^\/]+)\/', "", formatedPath)
727+
715728
# formatedPath = os.path.basename(req.form["file_path"]).replace("\\", "/").replace(filename, "")
716729
# if metadata["clinical_trial"] == "CHIRP":
717730
# saveFolderPath = self._processCHIRP(metadata, filename, uploadMetaData, filesSaved, fileTypeToPathMapping, formatedPath)
@@ -731,10 +744,8 @@ def acceptAndSaveFile(self, req:request):
731744
clinical_trial=metadata['clinical_trial'],
732745
test_centre=metadata["test_centre"],
733746
patient_trial_id=metadata["patient_trial_id"],
734-
fraction_name=metadata["fraction"],
735-
sub_fraction_name=metadata["sub_fraction"],
736747
centre_patient_no=int(metadata["centre_patient_no"])
737-
)
748+
) + formatedPath
738749
relativePath = relativeFolderPath + filename
739750
saveFolderPath = config.UPLOAD_FOLDER + '/' + relativeFolderPath
740751
filesSaved.append(relativePath)
@@ -743,6 +754,13 @@ def acceptAndSaveFile(self, req:request):
743754
for uploadedFileRecord in uploadMetaData["uploaded_files"]:
744755
if uploadedFileRecord["file_type"] == metadata["file_type"]:
745756
uploadedFileRecord["Files"].append(relativePath)
757+
if fractionNumber and fractionNumber not in uploadedFileRecord["fraction"]:
758+
uploadedFileRecord["fraction"].append(fractionNumber)
759+
uploadedFileRecord["sub_fraction"].append(fractionName)
760+
if fractionName and fractionName not in uploadedFileRecord["sub_fraction"]:
761+
uploadedFileRecord["sub_fraction"].append(fractionName)
762+
if saveFolderPath not in uploadedFileRecord["folder_path"]:
763+
uploadedFileRecord["folder_path"].append(saveFolderPath)
746764
filePathAppended = True
747765
break
748766

@@ -751,9 +769,10 @@ def acceptAndSaveFile(self, req:request):
751769
{
752770
"file_type": metadata["file_type"],
753771
"level": metadata["level"],
754-
"fraction": metadata["fraction"],
755-
"sub_fraction": metadata["sub_fraction"],
756-
"Files": [relativePath]
772+
"fraction": [fractionNumber] if fractionNumber else [],
773+
"sub_fraction": [fractionName] if fractionName else [],
774+
"Files": [relativePath],
775+
"folder_path": [saveFolderPath]
757776
}
758777
)
759778
print(f"saving {filename} in {saveFolderPath}")

0 commit comments

Comments
 (0)