Skip to content

Commit c498e2d

Browse files
QuyenLy87QuyenLy87
committed
RAP-48 Add a new parameter "RemoveRF2Files"
1 parent 6ce1b58 commit c498e2d

File tree

11 files changed

+103
-46
lines changed

11 files changed

+103
-46
lines changed

‎src/main/java/org/ihtsdo/rvf/core/service/MysqlValidationService.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.springframework.beans.factory.annotation.Autowired;
1414
import org.springframework.beans.factory.annotation.Value;
1515
import org.springframework.stereotype.Service;
16+
import org.springframework.util.CollectionUtils;
1617
import org.springframework.util.StringUtils;
1718

1819
import java.sql.SQLException;
@@ -54,20 +55,20 @@ public class MysqlValidationService {
5455

5556
private static final String RELEASE_TYPE_VALIDATION = "release-type-validation";
5657

57-
private final Set<String> legacyProspectiveVersions = new HashSet<>();
58+
private final Set<String> schemasToRemove = new HashSet<>();
5859

5960
private final ExecutorService executorService = Executors.newCachedThreadPool();
6061

6162
public ValidationStatusReport runRF2MysqlValidations(ValidationRunConfig validationConfig, ValidationStatusReport statusReport) throws BusinessServiceException, ExecutionException, InterruptedException {
6263
// Clean up the prospective databases if any
63-
for (String legacyProspectiveVersion : this.legacyProspectiveVersions) {
64-
this.releaseDataManager.dropSchema(legacyProspectiveVersion);
64+
for (String legacyVersion : this.schemasToRemove) {
65+
this.releaseDataManager.dropSchema(legacyVersion);
6566
}
66-
this.legacyProspectiveVersions.clear();
67+
this.schemasToRemove.clear();
6768
this.releaseDataManager.truncateQAResult();
6869

6970
MysqlExecutionConfig executionConfig = releaseVersionLoader.createExecutionConfig(validationConfig);
70-
this.legacyProspectiveVersions.add(executionConfig.getProspectiveVersion());
71+
this.schemasToRemove.add(executionConfig.getProspectiveVersion());
7172

7273
if (!StringUtils.hasLength(validationConfig.getPreviousRelease())) {
7374
executionConfig.setPreviousVersion(emptyRf2File);
@@ -84,14 +85,19 @@ public ValidationStatusReport runRF2MysqlValidations(ValidationRunConfig validat
8485
releaseVersionLoader.loadPreviousVersion(executionConfig);
8586
if (releaseVersionLoader.isUnknownVersion(executionConfig.getPreviousVersion())) {
8687
statusReport.addFailureMessage("Failed to load previous release " + executionConfig.getPreviousVersion());
88+
} else if (!CollectionUtils.isEmpty(executionConfig.getExcludedRF2Files())) {
89+
schemasToRemove.add(executionConfig.getPreviousVersion());
8790
}
8891

8992
// load dependency release
9093
releaseVersionLoader.loadDependencyVersion(executionConfig);
9194
lastItemLoadAttempted = "Dependency Release - " + executionConfig.getExtensionDependencyVersion();
9295
if (releaseVersionLoader.isUnknownVersion(executionConfig.getExtensionDependencyVersion())) {
9396
statusReport.addFailureMessage("Failed to load dependency release " + executionConfig.getExtensionDependencyVersion());
97+
} else if (!CollectionUtils.isEmpty(executionConfig.getExcludedRF2Files())) {
98+
schemasToRemove.add(executionConfig.getExtensionDependencyVersion());
9499
}
100+
95101
// load prospective version
96102
lastItemLoadAttempted = "Prospective Release - " + executionConfig.getProspectiveVersion();
97103
releaseVersionLoader.loadProspectiveVersion(statusReport, executionConfig, validationConfig);
@@ -142,7 +148,7 @@ private void runExtensionReleaseValidation(ValidationStatusReport statusReport,
142148
//loading international snapshot
143149
try {
144150
releaseVersionLoader.combineCurrentExtensionWithDependencySnapshot(executionConfig, validationConfig);
145-
this.legacyProspectiveVersions.add(executionConfig.getProspectiveVersion());
151+
this.schemasToRemove.add(executionConfig.getProspectiveVersion());
146152
} catch (BusinessServiceException e) {
147153
String errMsg = ExceptionUtils.getExceptionCause("Failed to prepare data for extension testing", e);
148154
statusReport.addFailureMessage(errMsg);

‎src/main/java/org/ihtsdo/rvf/core/service/ReleaseDataManager.java‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.ihtsdo.rvf.core.service;
22

3+
import jakarta.annotation.PostConstruct;
4+
import jakarta.annotation.Resource;
35
import org.apache.commons.dbcp.BasicDataSource;
46
import org.apache.commons.io.FileUtils;
57
import org.apache.commons.io.IOUtils;
@@ -20,8 +22,6 @@
2022
import org.springframework.core.io.ResourceLoader;
2123
import org.springframework.stereotype.Service;
2224

23-
import jakarta.annotation.PostConstruct;
24-
import jakarta.annotation.Resource;
2525
import java.io.*;
2626
import java.nio.file.Files;
2727
import java.nio.file.LinkOption;
@@ -150,7 +150,7 @@ public String getRVFVersion(String product, String releaseVersion) {
150150
* for uploading prospective releases since they do not need to be stored for later use.
151151
*
152152
*/
153-
public boolean uploadPublishedReleaseData(final InputStream inputStream, final String fileName, final String product, final String version) throws BusinessServiceException {
153+
public boolean uploadPublishedReleaseData(final InputStream inputStream, final String fileName, final String product, final String version, final List<String> excludedRF2Files) throws BusinessServiceException {
154154
// copy release pack zip to data location
155155
logger.info(RECEIVING_RELEASE_DATA_INFO_MSG, fileName);
156156
final File fileDestination = new File(sctDataFolder.getAbsolutePath(), fileName);
@@ -170,14 +170,14 @@ public boolean uploadPublishedReleaseData(final InputStream inputStream, final S
170170
logger.info("Release version is already known in RVF and the existing one will be deleted and reloaded: {}", rvfVersion);
171171
}
172172
List<String> rf2FilesLoaded = new ArrayList<>();
173-
String schemaName = loadSnomedData(rvfVersion, rf2FilesLoaded, fileDestination);
173+
String schemaName = loadSnomedData(rvfVersion, rf2FilesLoaded, excludedRF2Files, fileDestination);
174174
logger.info("schemaName = {}", schemaName);
175175
schemaNames.add(schemaName);
176176
return true;
177177
}
178178

179179

180-
public boolean uploadReleaseDataIntoDB(final InputStream inputStream, String fileName, String schemaName) throws BusinessServiceException {
180+
public boolean uploadReleaseDataIntoDB(final InputStream inputStream, String fileName, String schemaName, List<String> excludedRF2Files) throws BusinessServiceException {
181181
// copy release pack zip to data location
182182
logger.info(RECEIVING_RELEASE_DATA_INFO_MSG, fileName);
183183
final File fileDestination = new File(sctDataFolder.getAbsolutePath(), fileName);
@@ -197,22 +197,22 @@ public boolean uploadReleaseDataIntoDB(final InputStream inputStream, String fil
197197
}
198198
logger.info("Loading data into schema {}", schemaName);
199199
List<String> rf2FilesLoaded = new ArrayList<>();
200-
loadSnomedData(schemaName, rf2FilesLoaded, fileDestination);
200+
loadSnomedData(schemaName, rf2FilesLoaded, excludedRF2Files, fileDestination);
201201
logger.info("schemaName = {}", schemaName);
202202
return true;
203203
}
204204

205-
public boolean uploadPublishedReleaseData(final File releasePackZip, final String product, final String version) throws BusinessServiceException {
205+
public boolean uploadPublishedReleaseData(final File releasePackZip, final String product, final String version, final List<String> excludedRF2Files) throws BusinessServiceException {
206206
boolean result = false;
207207
try(InputStream inputStream = new FileInputStream(releasePackZip)) {
208-
result = uploadPublishedReleaseData(inputStream, releasePackZip.getName(), product, version);
208+
result = uploadPublishedReleaseData(inputStream, releasePackZip.getName(), product, version, excludedRF2Files);
209209
} catch (final IOException e) {
210210
logger.error("Error during upload release:" + releasePackZip.getName(), e);
211211
}
212212
return result;
213213
}
214214

215-
public String loadSnomedData(final String versionName, List<String> rf2FilesLoaded, final File... zipDataFile) throws BusinessServiceException {
215+
public String loadSnomedData(final String versionName, List<String> rf2FilesLoaded, List<String> excludedRF2Files, final File... zipDataFile) throws BusinessServiceException {
216216
File outputFolder = null;
217217
final String createdSchemaName = versionName.startsWith(RVF_DB_PREFIX) ? versionName : RVF_DB_PREFIX + versionName;
218218
final long startTime = Calendar.getInstance().getTimeInMillis();
@@ -232,7 +232,7 @@ public String loadSnomedData(final String versionName, List<String> rf2FilesLoad
232232
ZipFileUtils.extractFilesFromZipToOneFolder(zipFile, outputFolder.getAbsolutePath());
233233
}
234234
createSchema(createdSchemaName);
235-
loadReleaseFilesToDB(outputFolder, rvfDynamicDataSource, rf2FilesLoaded, createdSchemaName);
235+
loadReleaseFilesToDB(outputFolder, rvfDynamicDataSource, rf2FilesLoaded, createdSchemaName, excludedRF2Files);
236236
} catch (final RVFExecutionException | IOException e) {
237237
List<String> fileNames = Arrays.stream(zipDataFile).map(File::getName).toList();
238238
final String errorMsg = String.format("Error while loading file %s into version %s", Arrays.toString(fileNames.toArray()), versionName);
@@ -245,10 +245,18 @@ public String loadSnomedData(final String versionName, List<String> rf2FilesLoad
245245
return createdSchemaName;
246246
}
247247

248-
private void loadReleaseFilesToDB(final File rf2TextFilesDir, final RvfDynamicDataSource dataSource, List<String> rf2FilesLoaded, String schemaName) throws RVFExecutionException {
248+
private void loadReleaseFilesToDB(final File rf2TextFilesDir, final RvfDynamicDataSource dataSource, List<String> rf2FilesLoaded, String schemaName, List<String> excludedRF2Files) throws RVFExecutionException {
249249
if (rf2TextFilesDir != null) {
250+
final List<String> allExcludedRF2Files = new ArrayList<>();
251+
if (excludedRF2Files != null) {
252+
allExcludedRF2Files.addAll(excludedRF2Files);
253+
for (String excludedFile : excludedRF2Files) {
254+
allExcludedRF2Files.add(excludedFile.replace("Delta_", "Full_").replace("Delta-", "Full-"));
255+
allExcludedRF2Files.add(excludedFile.replace("Delta_", "Snapshot_").replace("Delta-", "Snapshot-"));
256+
}
257+
}
250258
final String[] rf2Files = rf2TextFilesDir.list((dir, name) ->
251-
name.endsWith(".txt") && (name.startsWith("der2") || name.startsWith("sct2") || name.startsWith("xder2") || name.startsWith("xsct2"))
259+
name.endsWith(".txt") && (name.startsWith("der2") || name.startsWith("sct2") || name.startsWith("xder2") || name.startsWith("xsct2")) && !isExcludedFile(name, allExcludedRF2Files)
252260
);
253261
if (rf2Files != null && rf2Files.length > 0) {
254262
final ReleaseFileDataLoader dataLoader = new ReleaseFileDataLoader(dataSource, schemaName, new MySqlDataTypeConverter());
@@ -257,6 +265,16 @@ private void loadReleaseFilesToDB(final File rf2TextFilesDir, final RvfDynamicDa
257265
}
258266
}
259267

268+
private boolean isExcludedFile(String name, List<String> allExcludedRF2Files) {
269+
if (allExcludedRF2Files == null || allExcludedRF2Files.isEmpty()) return false;
270+
String cleanFileName = name;
271+
if (cleanFileName.startsWith("x")) {
272+
cleanFileName = name.substring(1);
273+
}
274+
String filenameToCheck = cleanFileName.replace("sct2", "rel2").replace("der2", "rel2");
275+
return allExcludedRF2Files.stream().anyMatch(excludedRF2File -> filenameToCheck.contains(excludedRF2File.replaceAll("_\\d{8}.txt$", "")));
276+
}
277+
260278
public boolean isKnownRelease(String releaseVersion) {
261279
if (releaseVersion == null || releaseVersion.endsWith(ZIP_FILE_EXTENSION)) {
262280
return false;
@@ -594,12 +612,12 @@ public boolean restoreReleaseFromBinaryArchive(String schemaName) throws IOExcep
594612
}
595613
}
596614

597-
public boolean uploadPublishedReleaseFromStore(String releaseFilename, String schemaName) throws BusinessServiceException {
615+
public boolean uploadPublishedReleaseFromStore(String releaseFilename, String schemaName, List<String> excludedRF2Files) throws BusinessServiceException {
598616
// check local disk first
599617
if (!releaseStorageConfig.isUseCloud()) {
600618
File uploadedFile = new File(sctDataFolder, releaseFilename);
601619
if (uploadedFile.exists() && uploadedFile.canRead()) {
602-
loadSnomedData(schemaName, new ArrayList<>(), uploadedFile);
620+
loadSnomedData(schemaName, new ArrayList<>(), excludedRF2Files, uploadedFile);
603621
return true;
604622
}
605623
}
@@ -610,7 +628,7 @@ public boolean uploadPublishedReleaseFromStore(String releaseFilename, String sc
610628
} catch (IOException e) {
611629
throw new BusinessServiceException("Failed to read file " + releaseFilename + " via " + releaseStorageConfig.toString(), e);
612630
}
613-
uploadReleaseDataIntoDB(inputStream, releaseFilename, schemaName);
631+
uploadReleaseDataIntoDB(inputStream, releaseFilename, schemaName, excludedRF2Files);
614632
return true;
615633
}
616634

0 commit comments

Comments
 (0)