From 23c7dc56bc084fe460e3f74149bf7cec84ec8338 Mon Sep 17 00:00:00 2001 From: Senjo Jose Date: Wed, 14 Jun 2023 16:13:52 +1000 Subject: [PATCH 01/11] Support functions with sql statements --- .../org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java index fbf62e12b..2abe06531 100644 --- a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java +++ b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java @@ -38,6 +38,7 @@ public class AssertionsDatabaseImporter { private static final String CREATE_PROCEDURE = "CREATE PROCEDURE"; + private static final String CREATE_FUNCTION = "CREATE FUNCTION"; private static final Logger logger = LoggerFactory.getLogger(AssertionsDatabaseImporter.class); private static final String RESOURCE_PATH_SEPARATOR = "/"; @@ -153,7 +154,7 @@ public void addSqlTestToAssertion(final Assertion assertion, final String sql){ for (final StatementSplitter.Statement statement : splitter.getCompleteStatements()) { String cleanedSql = statement.statement(); logger.debug("sql to be cleaned:" + cleanedSql); - if ( cleanedSql.startsWith(CREATE_PROCEDURE) || cleanedSql.startsWith(CREATE_PROCEDURE.toLowerCase())) { + if( cleanedSql.toUpperCase().startsWith(CREATE_PROCEDURE) || cleanedSql.toUpperCase().startsWith(CREATE_FUNCTION)) { storedProcedureFound = true; } // Process SQL statement From eeed8b2e308c2654d815a243cd68348367134014 Mon Sep 17 00:00:00 2001 From: Senjo Jose Date: Fri, 16 Jun 2023 17:17:17 +1000 Subject: [PATCH 02/11] Added support for pre-requisites sql --- .../service/AssertionExecutionService.java | 65 +---- .../service/util/MySqlQueryTransformer.java | 111 +++++++++ .../importer/AssertionsDatabaseImporter.java | 235 +++++++++++++----- 3 files changed, 295 insertions(+), 116 deletions(-) create mode 100644 src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java diff --git a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java index 0e42697e9..209671416 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java @@ -1,24 +1,27 @@ package org.ihtsdo.rvf.core.service; +import java.sql.*; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import javax.naming.ConfigurationException; import org.apache.commons.dbcp.BasicDataSource; import org.ihtsdo.rvf.core.data.model.*; import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; -import org.ihtsdo.rvf.importer.AssertionGroupImporter.ProductName; +import org.ihtsdo.rvf.core.service.util.MySqlQueryTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; import jakarta.annotation.Resource; -import javax.naming.ConfigurationException; -import java.sql.*; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.concurrent.*; -import java.util.regex.Pattern; @Service @@ -221,52 +224,10 @@ private String[] splitCommand(ExecutionCommand command) { } private List transformSql(String[] parts, Assertion assertion, MysqlExecutionConfig config) throws ConfigurationException { - List result = new ArrayList<>(); - String defaultCatalog = dataSource.getDefaultCatalog(); - String prospectiveSchema = config.getProspectiveVersion(); - final String[] nameParts = config.getProspectiveVersion().split("_"); - String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); - String includedModules = String.join(",", config.getIncludedModules()); - String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); - - String previousReleaseSchema = config.getPreviousVersion(); - String dependencyReleaseSchema = config.getExtensionDependencyVersion(); - validateSchemas(config, prospectiveSchema, previousReleaseSchema); - - for( String part : parts) { - if ((part.contains("") && previousReleaseSchema == null) - || (part.contains("") && dependencyReleaseSchema == null)) { - continue; - } - - logger.debug("Original sql statement: {}", part); - final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL); - part = commentPattern.matcher(part).replaceAll(""); - // replace all substitutions for exec - part = part.replaceAll("", String.valueOf(config.getExecutionId())); - part = part.replaceAll("", String.valueOf(assertion.getAssertionId())); - part = part.replaceAll("", defaultModuleId); - part = part.replaceAll("", includedModules); - part = part.replaceAll("", version); - // watch out for any 's that users might have introduced - part = part.replaceAll("qa_result", defaultCatalog+ "." + qaResulTableName); - part = part.replaceAll("", prospectiveSchema); - part = part.replaceAll("", prospectiveSchema); - part = part.replaceAll("", internationalModules); - if (previousReleaseSchema != null) { - part = part.replaceAll("", previousReleaseSchema); - } - if (dependencyReleaseSchema != null) { - part = part.replaceAll("", dependencyReleaseSchema); - } - part = part.replaceAll("", deltaTableSuffix); - part = part.replaceAll("", snapshotTableSuffix); - part = part.replaceAll("", fullTableSuffix); - part = part.trim(); - logger.debug("Transformed sql statement: {}", part); - result.add(part); - } - return result; + String qaResult = dataSource.getDefaultCatalog()+ "." + qaResulTableName; + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + Map configMap = Map.of("qa_result",qaResult, "", String.valueOf(assertion.getAssertionId())); + return queryTransformer.transformSql(parts, config, configMap); } private static void validateSchemas(MysqlExecutionConfig config, String prospectiveSchema, String previousReleaseSchema) throws ConfigurationException { diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java new file mode 100644 index 000000000..6dd49f832 --- /dev/null +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -0,0 +1,111 @@ +package org.ihtsdo.rvf.core.service.util; + +import com.facebook.presto.sql.parser.StatementSplitter; +import com.google.common.collect.ImmutableSet; +import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; +import org.ihtsdo.rvf.importer.AssertionGroupImporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.naming.ConfigurationException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class MySqlQueryTransformer { + private final Logger logger = LoggerFactory.getLogger(MySqlQueryTransformer.class); + + private static final String FAILED_TO_FIND_RVF_DB_SCHEMA = "Failed to find rvf db schema for "; + + private String deltaTableSuffix = "d"; + private String snapshotTableSuffix = "s"; + private String fullTableSuffix = "f"; + private static final String DEFAULT_DELIMITER = ";"; + private static final String DELIMITER_REGEX_PATTERN = "^[ ]*(delimiter|DELIMITER)"; + + public List transformSql(String[] parts, MysqlExecutionConfig config, final Map configMap) throws ConfigurationException { + List result = new ArrayList<>(); + String prospectiveSchema = config.getProspectiveVersion(); + final String[] nameParts = config.getProspectiveVersion().split("_"); + String moduleId = (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); + String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); + + String previousReleaseSchema = config.getPreviousVersion(); + String dependencyReleaseSchema = config.getExtensionDependencyVersion(); + + //We need both these schemas to exist + if (prospectiveSchema == null) { + throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema); + } + + if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) { + throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); + } + for( String part : parts) { + if ((part.contains("") && previousReleaseSchema == null) + || (part.contains("") && dependencyReleaseSchema == null)) { + continue; + } + + logger.debug("Original sql statement: {}", part); + // remove all SQL comments - //TODO might throw errors for -- style comments + final Pattern commentPattern = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL); + part = commentPattern.matcher(part).replaceAll(""); + // replace all substitutions for exec + part = part.replaceAll("", String.valueOf(config.getExecutionId())); + part = part.replaceAll("", moduleId); + part = part.replaceAll("", version); + // watch out for any 's that users might have introduced + part = part.replaceAll("", prospectiveSchema); + part = part.replaceAll("", prospectiveSchema); + if (previousReleaseSchema != null) { + part = part.replaceAll("", previousReleaseSchema); + } + if (dependencyReleaseSchema != null) { + part = part.replaceAll("", dependencyReleaseSchema); + } + part = part.replaceAll("", deltaTableSuffix); + part = part.replaceAll("", snapshotTableSuffix); + part = part.replaceAll("", fullTableSuffix); + for(Map.Entry configMapEntry: configMap.entrySet()){ + part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); + } + part.trim(); + logger.debug("Transformed sql statement: {}", part); + result.add(part); + } + return result; + } + /** + * Convert given sql file content to multiple statements + * @param sqlFileContent + * @return + */ + public List transformToStatements(String sqlFileContent){ + String delimiter = DEFAULT_DELIMITER; + List result = new ArrayList<>(); + String[] sqlChunks = sqlFileContent.trim().split(DELIMITER_REGEX_PATTERN, Pattern.MULTILINE); + for (int i = 0; i < sqlChunks.length; i++) { + String sqlChunk = sqlChunks[i].trim(); + if (!sqlChunk.isEmpty()) { + if (i > 0) { + delimiter = sqlChunk.trim().replaceAll("(?s)^([^ \r\n]+).*$", "$1"); + sqlChunk = sqlChunk.trim().replaceAll("(?s)^[^ \r\n]+(.*)$", "$1").trim(); + } + if (!sqlChunk.isEmpty()) { + logger.debug("Executing pre-requisite SQL: " + sqlChunk); + final StatementSplitter splitter = new StatementSplitter(sqlChunk, ImmutableSet.of(delimiter)); + if (splitter.getCompleteStatements() == null || splitter.getCompleteStatements().isEmpty()) { + logger.warn(String.format("SQL statements not ending with %s %s",delimiter, sqlChunk) ); + } + result= splitter.getCompleteStatements().stream().map(s -> s.statement()).collect(Collectors.toList()); + + } + } + + } + return result; + } +} diff --git a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java index 2abe06531..048b8a1dd 100644 --- a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java +++ b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java @@ -8,6 +8,7 @@ import org.ihtsdo.rvf.core.data.model.Test; import org.ihtsdo.rvf.core.data.model.TestType; import org.ihtsdo.rvf.core.service.AssertionService; +import org.ihtsdo.rvf.core.service.util.MySqlQueryTransformer; import org.jdom2.Document; import org.jdom2.Element; import org.jdom2.JDOMException; @@ -28,6 +29,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +/** /** * An implementation of a {@link org.ihtsdo.rvf.importer.AssertionsDatabaseImporter} that imports older * Release Assertion Toolkit content via XML and SQL files. The XML file defines assertions and SQL files are @@ -54,82 +56,180 @@ public boolean isAssertionImportRequired() { return assertions == null || assertions.isEmpty(); } - /** - * Imports assertions from the given manifest file input stream. The manifest file should be an XML file - * containing a list of assertions to import. Each assertion should contain a list of SQL scripts to execute - * to populate the assertion tests. - * @param manifestInputStream the manifest file input stream - * @param sqlFileDir the directory containing the SQL files - * @throws IOException if there is an error reading the manifest file - */ - public void importAssertionsFromManifest(final InputStream manifestInputStream, final String sqlFileDir) throws IOException { - final List scriptElements = getScriptElements(manifestInputStream); - if (scriptElements.isEmpty()) { - logger.warn("There are no script elements to import in the XML file provided. Please note that the " + - "XML file should contain element named script"); - } else { - // Create Assertions and tests from script elements - for (final Element element : scriptElements) { - createAssertionAndTest(element, sqlFileDir); + public List getAssertionsFromFile(final InputStream manifestInputStream) { + final Document xmlDocument = getJDomDocumentFromFile(manifestInputStream); + List assertions = new ArrayList<>(); + if(xmlDocument != null) + { + final XPathFactory factory = XPathFactory.instance(); + final XPathExpression expression = factory.compile("//script"); + final List scriptElements = expression.evaluate(xmlDocument); + if(scriptElements.size() > 0) { + // get various values from script element + for (final Element element : scriptElements) { + assertions.add(createAssertionFromElement(element)); + + } } } + return assertions; } - List getScriptElements(InputStream manifestInputStream) throws IOException { - // Get JDOM document from given manifest file input stream - final Document xmlDocument; - try { - final SAXBuilder sax = new SAXBuilder(); - xmlDocument = sax.build(manifestInputStream); - } catch (JDOMException e) { - throw new IOException("Failed to parse manifest file.", e); - } - final XPathFactory factory = XPathFactory.instance(); - final XPathExpression expression = factory.compile("//script", new ElementFilter("script")); - return expression.evaluate(xmlDocument); - } + public void importAssertionsFromFile(final InputStream manifestInputStream, final String sqlResourcesFolderLocation){ + // get JDOM document from given manifest file + final Document xmlDocument = getJDomDocumentFromFile(manifestInputStream); + if(xmlDocument != null) + { + final XPathFactory factory = XPathFactory.instance(); + final XPathExpression expression = factory.compile("//script"); + final List scriptElements = expression.evaluate(xmlDocument); + if(scriptElements.size() > 0) + { + // get various values from script element + for(final Element element : scriptElements) + { + final Assertion assertion = createAssertionFromElement(element); + if(assertion != null){ + try + { + logger.info("Created assertion id : " + assertion.getAssertionId()); + assert UUID.fromString(element.getAttributeValue("uuid")).equals(assertion.getUuid()); + // get Sql file name from element and use it to add SQL test + final String sqlFileName = element.getAttributeValue("sqlFile"); + logger.info("sqlFileName = " + sqlFileName); + String category = element.getAttributeValue("category"); + /* + We know category is written as file-centric-validation, component-centric-validation, etc + We use this to generate the corresponding using folder name = category - validation + */ + final int index = category.indexOf("validation"); + if(index > -1) + { + category = category.substring(0, index-1); + } + logger.info("category = " + category); + String sqlResourceFileName = sqlResourcesFolderLocation + RESOURCE_PATH_SEPARATOR + category + RESOURCE_PATH_SEPARATOR + sqlFileName; + InputStream sqlInputStream = null; + sqlInputStream = assertionResourceManager.readResourceStream(sqlResourceFileName); + if (sqlInputStream != null) { + final String sqlString = readStream(sqlInputStream); + // add test to assertion + if(isPreRequisitesSql(sqlFileName)){ //handle pre-requisites.sql + addPreRequisiteSqlToAssertion(assertion, sqlString); + } else{ + addSqlTestToAssertion(assertion, sqlString); + } + } else { + String msg = "Failed to find sql file name from source:" + sqlResourceFileName + " for assertion uuid:" + assertion.getUuid(); + logger.error(msg); + throw new IllegalStateException(msg); + } + } + catch (final Exception e) { + logger.warn("Error reading sql from input stream. Nested exception is : " + e.getMessage()); + throw new IllegalStateException("Failed to add sql script test to assertion with uuid:" + assertion.getUuid(), e); + } + } + else{ + throw new IllegalStateException("Error creating assertion"); + } + } - private void createAssertionAndTest(Element element, String sqlFileDir) { - Assertion assertion = createAssertionFromElement(element); - // Persist assertion - assertionService.create(assertion); - // Add SQL tests to assertion - try { - addSqlTestsToAssertion(assertion, element, sqlFileDir); - } catch (Exception e) { - String errorMsg = "Failed to add sql script test to assertion with uuid:" + assertion.getUuid(); - logger.error(errorMsg, e); - throw new IllegalStateException(errorMsg, e); + // finally print all lookup map contents for debugging - //todo save somewhere? + logger.debug("lookupMap = " + lookupMap); + } + else{ + logger.error("There are no script elements to import in the XML file provided. Please note that the " + + "XML file should contain element named script"); + } + } + else{ + logger.warn("Error generating document from xml file passed : " + manifestInputStream); } } - private void addSqlTestsToAssertion(Assertion assertion, Element element, String sqlFileDir) throws IOException { - assert UUID.fromString(element.getAttributeValue("uuid")).equals(assertion.getUuid()); - // get Sql file name from element and use it to add SQL test - final String sqlFileName = element.getAttributeValue("sqlFile"); - logger.debug("sqlFileName = " + sqlFileName); - String category = element.getAttributeValue("category"); - // Category is written as file-centric-validation, component-centric-validation, etc. - // Use this to generate the corresponding using folder name = category - validation - final int index = category.indexOf("validation"); - if (index > -1) { - category = category.substring(0, index-1); - } - logger.debug("category = {} ", category); - String sqlFullFilename = sqlFileDir + RESOURCE_PATH_SEPARATOR + category + RESOURCE_PATH_SEPARATOR + sqlFileName; - InputStream sqlInputStream = assertionResourceManager.readResourceStream(sqlFullFilename); - if (sqlInputStream == null) { - String msg = "Failed to find sql file name from source:" + sqlFullFilename + " for assertion uuid:" + assertion.getUuid(); - logger.error(msg); - throw new IllegalStateException(msg); + /** + * Imports assertions from the given manifest file input stream. The manifest file should be an XML file + * containing a list of assertions to import. Each assertion should contain a list of SQL scripts to execute + * to populate the assertion tests. + * @param manifestInputStream the manifest file input stream + * @param sqlFileDir the directory containing the SQL files + * @throws IOException if there is an error reading the manifest file + */ + public void importAssertionsFromManifest(final InputStream manifestInputStream, final String sqlFileDir) throws IOException { + final List scriptElements = getScriptElements(manifestInputStream); + if (scriptElements.isEmpty()) { + logger.warn("There are no script elements to import in the XML file provided. Please note that the " + + "XML file should contain element named script"); + } else { + // Create Assertions and tests from script elements + for (final Element element : scriptElements) { + createAssertionAndTest(element, sqlFileDir); } - final String sqlString = IOUtils.toString(sqlInputStream, UTF_8); - // add test to assertion - addSqlTestToAssertion(assertion, sqlString); } + } + + List getScriptElements(InputStream manifestInputStream) throws IOException { + // Get JDOM document from given manifest file input stream + final Document xmlDocument; + try { + final SAXBuilder sax = new SAXBuilder(); + xmlDocument = sax.build(manifestInputStream); + } catch (JDOMException e) { + throw new IOException("Failed to parse manifest file.", e); + } + final XPathFactory factory = XPathFactory.instance(); + final XPathExpression expression = factory.compile("//script", new ElementFilter("script")); + return expression.evaluate(xmlDocument); + } + + + private void createAssertionAndTest(Element element, String sqlFileDir) { + Assertion assertion = createAssertionFromElement(element); + // Persist assertion + assertionService.create(assertion); + // Add SQL tests to assertion + try { + addSqlTestsToAssertion(assertion, element, sqlFileDir); + } catch (Exception e) { + String errorMsg = "Failed to add sql script test to assertion with uuid:" + assertion.getUuid(); + logger.error(errorMsg, e); + throw new IllegalStateException(errorMsg, e); + } + } + + private void addSqlTestsToAssertion(Assertion assertion, Element element, String sqlFileDir) throws IOException { + assert UUID.fromString(element.getAttributeValue("uuid")).equals(assertion.getUuid()); + // get Sql file name from element and use it to add SQL test + final String sqlFileName = element.getAttributeValue("sqlFile"); + logger.debug("sqlFileName = " + sqlFileName); + String category = element.getAttributeValue("category"); + // Category is written as file-centric-validation, component-centric-validation, etc. + // Use this to generate the corresponding using folder name = category - validation + final int index = category.indexOf("validation"); + if (index > -1) { + category = category.substring(0, index-1); + } + logger.debug("category = {} ", category); + String sqlFullFilename = sqlFileDir + RESOURCE_PATH_SEPARATOR + category + RESOURCE_PATH_SEPARATOR + sqlFileName; + InputStream sqlInputStream = assertionResourceManager.readResourceStream(sqlFullFilename); + if (sqlInputStream == null) { + String msg = "Failed to find sql file name from source:" + sqlFullFilename + " for assertion uuid:" + assertion.getUuid(); + logger.error(msg); + throw new IllegalStateException(msg); + } + final String sqlString = IOUtils.toString(sqlInputStream, UTF_8); + // add test to assertion + addSqlTestToAssertion(assertion, sqlString); + } + + private boolean isPreRequisitesSql(String sqlFileName){ + return sqlFileName.equalsIgnoreCase("pre-requisites.sql"); + } + + private Assertion createAssertionFromElement(final Element element){ - private Assertion createAssertionFromElement(final Element element){ final String category = element.getAttributeValue("category"); final String uuid = element.getAttributeValue("uuid"); final String text = element.getAttributeValue("text"); @@ -211,6 +311,13 @@ private void uploadTest (Assertion assertion, String originalSql, List s assertionService.addTests(assertion, tests); } + protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) + throws RuntimeException { + MySqlQueryTransformer mySqlQueryTransformer = new MySqlQueryTransformer(); + final List sqlStatements = mySqlQueryTransformer.transformToStatements(preRequisiteSql); + uploadTest(assertion, preRequisiteSql, sqlStatements); + } + private Map getRvfSchemaMapping(String ratSchema){ String rvfSchema = ""; ratSchema = ratSchema.trim(); From 23f9599db79c47fa2529ea0f84beeec80476718e Mon Sep 17 00:00:00 2001 From: Senjo Jose Date: Mon, 19 Jun 2023 11:36:23 +1000 Subject: [PATCH 03/11] Added support for pre-requisites sql --- .../service/util/MySqlQueryTransformer.java | 14 +- .../importer/AssertionsDatabaseImporter.java | 17 +- .../util/MySqlQueryTransformerTest.java | 167 ++++++++++++++++++ 3 files changed, 184 insertions(+), 14 deletions(-) create mode 100644 src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index 6dd49f832..25c69ef17 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -2,6 +2,7 @@ import com.facebook.presto.sql.parser.StatementSplitter; import com.google.common.collect.ImmutableSet; +import org.ihtsdo.otf.rest.exception.BusinessServiceException; import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; import org.ihtsdo.rvf.importer.AssertionGroupImporter; import org.slf4j.Logger; @@ -28,6 +29,9 @@ public class MySqlQueryTransformer { public List transformSql(String[] parts, MysqlExecutionConfig config, final Map configMap) throws ConfigurationException { List result = new ArrayList<>(); String prospectiveSchema = config.getProspectiveVersion(); + if (prospectiveSchema == null) { + throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema); + } final String[] nameParts = config.getProspectiveVersion().split("_"); String moduleId = (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); @@ -36,10 +40,6 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi String dependencyReleaseSchema = config.getExtensionDependencyVersion(); //We need both these schemas to exist - if (prospectiveSchema == null) { - throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema); - } - if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) { throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); } @@ -83,7 +83,7 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi * @param sqlFileContent * @return */ - public List transformToStatements(String sqlFileContent){ + public List transformToStatements(String sqlFileContent) throws BusinessServiceException { String delimiter = DEFAULT_DELIMITER; List result = new ArrayList<>(); String[] sqlChunks = sqlFileContent.trim().split(DELIMITER_REGEX_PATTERN, Pattern.MULTILINE); @@ -98,7 +98,9 @@ public List transformToStatements(String sqlFileContent){ logger.debug("Executing pre-requisite SQL: " + sqlChunk); final StatementSplitter splitter = new StatementSplitter(sqlChunk, ImmutableSet.of(delimiter)); if (splitter.getCompleteStatements() == null || splitter.getCompleteStatements().isEmpty()) { - logger.warn(String.format("SQL statements not ending with %s %s",delimiter, sqlChunk) ); + String errorMsg = String.format("SQL statements not ending with %s %s",delimiter, sqlChunk); + logger.error( errorMsg); + throw new BusinessServiceException(errorMsg); } result= splitter.getCompleteStatements().stream().map(s -> s.statement()).collect(Collectors.toList()); diff --git a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java index 048b8a1dd..423d69e53 100644 --- a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java +++ b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java @@ -3,6 +3,7 @@ import com.facebook.presto.sql.parser.StatementSplitter; import org.apache.commons.io.IOUtils; import org.ihtsdo.otf.resourcemanager.ResourceManager; +import org.ihtsdo.otf.rest.exception.BusinessServiceException; import org.ihtsdo.rvf.core.data.model.Assertion; import org.ihtsdo.rvf.core.data.model.ExecutionCommand; import org.ihtsdo.rvf.core.data.model.Test; @@ -296,7 +297,14 @@ public void addSqlTestToAssertion(final Assertion assertion, final String sql){ uploadTest(assertion, sql, statements); } - private void uploadTest (Assertion assertion, String originalSql, List sqlStatements) { + protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) + throws RuntimeException { + MySqlQueryTransformer mySqlQueryTransformer = new MySqlQueryTransformer(); + final List sqlStatements = mySqlQueryTransformer.transformToStatements(preRequisiteSql); + uploadTest(assertion, preRequisiteSql, sqlStatements); + } + + private void uploadTest (Assertion assertion, String originalSql, List sqlStatements) { final ExecutionCommand command = new ExecutionCommand(); command.setTemplate(originalSql); command.setStatements(sqlStatements); @@ -311,13 +319,6 @@ private void uploadTest (Assertion assertion, String originalSql, List s assertionService.addTests(assertion, tests); } - protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) - throws RuntimeException { - MySqlQueryTransformer mySqlQueryTransformer = new MySqlQueryTransformer(); - final List sqlStatements = mySqlQueryTransformer.transformToStatements(preRequisiteSql); - uploadTest(assertion, preRequisiteSql, sqlStatements); - } - private Map getRvfSchemaMapping(String ratSchema){ String rvfSchema = ""; ratSchema = ratSchema.trim(); diff --git a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java new file mode 100644 index 000000000..e04dea664 --- /dev/null +++ b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java @@ -0,0 +1,167 @@ +package org.ihtsdo.rvf.core.service.util; + +import org.ihtsdo.otf.rest.exception.BusinessServiceException; +import org.ihtsdo.rvf.core.service.config.MysqlExecutionConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import javax.naming.ConfigurationException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + + +@ExtendWith(MockitoExtension.class) +public class MySqlQueryTransformerTest { + @Mock + MysqlExecutionConfig config; + @Test + public void transformToStatements() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER //\n" + + "SET default_storage_engine=MYISAM//\n" + + "DROP TABLE IF EXISTS concept_active//\n" + + "CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1//\n" + + "create unique index concept_active_id_ix on concept_active(id)//\n" + + "create index concept_active_effectivetime_ix on concept_active(effectivetime)//\n" + + "create index concept_active_definitionstatusid_ix on concept_active(definitionstatusid)//\n" + + "create index concept_active_moduleid_ix on concept_active(moduleid)//\n" + + "create index concept_active_active_ix on concept_active(active)//"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(8, result.size()); + assertEquals("create index concept_active_active_ix on concept_active(active)", result.get(7)); + assertEquals("CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1", result.get(2)); + } + @Test + public void transformToStatementsWithSqlFunctions() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER // drop function if exists get_cr_ADRS_PT//\n" + + "create function get_cr_ADRS_PT(candidate bigint) returns varchar(4500)\n" + + "BEGIN RETURN (SELECT da.term FROM langrefset_active as lr join description_active as da\n" + + " on da.id = lr.referencedcomponentid WHERE lr.acceptabilityid = 900000000000548007 and conceptid = candidate); END//\n" + + "\n" + + "drop function if exists get_cr_FSN//\n" + + "create function get_cr_FSN(candidate bigint) returns varchar(4500)\n" + + "BEGIN RETURN (SELECT term FROM description_active where conceptId = candidate and typeId = 900000000000003001 and languageCode = 'en'); END//\n" + + "\n" + + "drop function if exists get_cr_PercentDefined//\n" + + "create function get_cr_PercentDefined(refset bigint) returns decimal(6, 4)\n" + + "BEGIN SET @refset = refset; SET @refsetSize = (select count(1) from simplerefset_active where refsetId = @refset); " + + "SET @definedCount = (select count(1) from concept_active where definitionStatusId = 900000000000073002 " + + "and id in (select referencedComponentId from simplerefset_active where refsetId = @refset)); " + + "RETURN CONVERT(@definedCount/ @refsetSize * 100,DECIMAL(6,4)); END//\n"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(6, result.size()); + assertTrue(result.get(0).startsWith("drop function if exists get_cr_ADRS_PT") ); + assertTrue(result.get(5).startsWith("create function get_cr_PercentDefined(refset bigint)")); + } + @Test + public void transformToStatementsUsingDefaultDelimiter() throws BusinessServiceException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;\n" + + "CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1;\n" + + "create unique index concept_active_id_ix on concept_active(id);\n" + + "create index concept_active_effectivetime_ix on concept_active(effectivetime);\n" + + "create index concept_active_definitionstatusid_ix on concept_active(definitionstatusid);\n" + + "create index concept_active_moduleid_ix on concept_active(moduleid);\n" + + "create index concept_active_active_ix on concept_active(active);"; + List result = queryTransformer.transformToStatements(sqlToTest); + assertEquals(8, result.size()); + assertEquals("create index concept_active_active_ix on concept_active(active)", result.get(7)); + assertEquals("CREATE TABLE concept_active ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci AS\n" + + "SELECT * FROM concept_ AS concepts WHERE active = 1", result.get(2)); + } + @Test + public void transformToStatementInvalidDelimiter() { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "DELIMITER // SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;"; + BusinessServiceException exception = assertThrows(BusinessServiceException.class, () -> { + queryTransformer.transformToStatements(sqlToTest); + }); + + String expectedMessagePattern = "SQL statements not ending with // SET default_storage_engine=MYISAM"; + String actualMessage = exception.getMessage(); + assertTrue(actualMessage.startsWith(expectedMessagePattern)); + + } + + @Test + public void transformSql() throws ConfigurationException { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sql = "insert into res_concepts_edited\n" + + "\tselect distinct id\n" + + "\tfrom .concept_\n" + + "\twhere id in ( \n" + + "\t\tselect id from .concept_\n" + + " union \n" + + " select conceptid from .description_\n" + + " union \n" + + " select conceptid from .textdefinition_\n" + + " union \n" + + " select sourceid from .stated_relationship_\n" + + " union\n" + + " select referencedcomponentid from .owlexpressionrefset_\n" + + " union\n" + + " select b.conceptid from .langrefset_ a \n" + + " left join .description_ b on a.referencedcomponentid=b.id\n" + + " union \n" + + " select referencedcomponentid from .attributevaluerefset_\n" + + " where refsetid = '900000000000489007'\n" + + " union \n" + + " select a.conceptid from .description_ a\n" + + " join .attributevaluerefset_ b on a.id = b.referencedcomponentid\n" + + " where b.refsetid = '900000000000490003'\n" + + "\tunion\n" + + " select referencedcomponentid\n" + + " from .associationrefset_\n" + + " where refsetid in ('900000000000523009','900000000000526001','900000000000527005','900000000000530003','1186924009','1186921001')\n" + + "\tunion\n" + + " select a.conceptid from .description_ a \n" + + " join .associationrefset_ b on a.id = b.referencedcomponentid \n" + + " where b.refsetid = '900000000000531004'\n" + + "\tunion\n" + + " select referencedcomponentid, from .simplerefset_\n" + + "\tunion\n" + + " select referencedcomponentid from .simplemaprefset_\n" + + " )"; + String[] sqlParts = {sql}; + String testAssertionId = "xyz"; + String testQaResult = "abc"; + when(config.getProspectiveVersion()).thenReturn("rvf_au_20221231_230619110027"); + when(config.getPreviousVersion()).thenReturn(String.valueOf(10)); + Map configMap = Map.of("qa_result",testQaResult, "", String.valueOf(testAssertionId)); + List result = queryTransformer.transformSql(sqlParts,config, configMap); + assertEquals(1,result.size()); + assertFalse(result.get(0).contains("")); + assertFalse(result.get(0).contains("")); + assertTrue(result.get(0).contains("rvf_au_20221231_230619110027.description_d")); + assertTrue(result.get(0).contains(testAssertionId)); + } + + @Test + public void transformSqlMissingConfig() { + MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); + String sqlToTest = "SET default_storage_engine=MYISAM/;\n" + + "DROP TABLE IF EXISTS concept_active;"; + String[] sqlParts = {sqlToTest}; + ConfigurationException exception = assertThrows(ConfigurationException.class, () -> { + queryTransformer.transformSql(sqlParts,config, Collections.emptyMap()); + }); + + String expectedMessagePattern = "Failed to find rvf db schema for null"; + String actualMessage = exception.getMessage(); + assertTrue(actualMessage.startsWith(expectedMessagePattern)); + + } + +} \ No newline at end of file From 4c7d5907d18e349635b9f1995f33dcc29417f607 Mon Sep 17 00:00:00 2001 From: Dion McMurtrie Date: Thu, 14 Sep 2023 14:04:08 +1000 Subject: [PATCH 04/11] test fix --- .../rvf/core/service/util/MySqlQueryTransformer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index 25c69ef17..7348177b1 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -7,6 +7,7 @@ import org.ihtsdo.rvf.importer.AssertionGroupImporter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; import javax.naming.ConfigurationException; import java.util.ArrayList; @@ -34,7 +35,6 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi } final String[] nameParts = config.getProspectiveVersion().split("_"); String moduleId = (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); - String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); String previousReleaseSchema = config.getPreviousVersion(); String dependencyReleaseSchema = config.getExtensionDependencyVersion(); @@ -43,6 +43,11 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) { throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); } + + final String[] nameParts = config.getProspectiveVersion().split("_"); + String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); + String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(",")); + String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED"); for( String part : parts) { if ((part.contains("") && previousReleaseSchema == null) || (part.contains("") && dependencyReleaseSchema == null)) { From afbcb53506dc401a75fede7b2fcb8ad801ebb112 Mon Sep 17 00:00:00 2001 From: lukeswindale Date: Thu, 14 Sep 2023 14:30:37 +1000 Subject: [PATCH 05/11] Fixed merge error --- .../org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index 7348177b1..5ab86824c 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -74,6 +74,8 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi part = part.replaceAll("", deltaTableSuffix); part = part.replaceAll("", snapshotTableSuffix); part = part.replaceAll("", fullTableSuffix); + part = part.replaceAll(Pattern.quote("[[:<:]]"),"\\\\b" ); + part = part.replaceAll(Pattern.quote("[[:>:]]"),"\\\\b" ); for(Map.Entry configMapEntry: configMap.entrySet()){ part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); } From e38ce5ecbebaaed0f8a9a856049d1692c562bc82 Mon Sep 17 00:00:00 2001 From: Dion McMurtrie Date: Fri, 22 Sep 2023 16:05:38 +1000 Subject: [PATCH 06/11] shot at fixing over-replacement of the schema/table name --- .../rvf/core/service/util/MySqlQueryTransformer.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index 5ab86824c..d38a88cd5 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -63,6 +63,10 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi part = part.replaceAll("", moduleId); part = part.replaceAll("", version); // watch out for any 's that users might have introduced + part = part.replaceAll("qa_result", + configMap.get("qa_result").startsWith(configMap.get("default_catalog")) + ? configMap.get("default_catalog") + "." + configMap.get("qa_result") + : configMap.get("qa_result")); part = part.replaceAll("", prospectiveSchema); part = part.replaceAll("", prospectiveSchema); if (previousReleaseSchema != null) { @@ -77,7 +81,9 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi part = part.replaceAll(Pattern.quote("[[:<:]]"),"\\\\b" ); part = part.replaceAll(Pattern.quote("[[:>:]]"),"\\\\b" ); for(Map.Entry configMapEntry: configMap.entrySet()){ - part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); + if (!configMapEntry.getKey().equals("qa_result")) { + part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); + } } part.trim(); logger.debug("Transformed sql statement: {}", part); From 68439c3fd7b7adb4e880f9d175ca971978fb929c Mon Sep 17 00:00:00 2001 From: Dion McMurtrie Date: Fri, 22 Sep 2023 16:50:35 +1000 Subject: [PATCH 07/11] added logging for the config map --- .../ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index d38a88cd5..aa4d4bde0 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -27,7 +27,11 @@ public class MySqlQueryTransformer { private static final String DEFAULT_DELIMITER = ";"; private static final String DELIMITER_REGEX_PATTERN = "^[ ]*(delimiter|DELIMITER)"; - public List transformSql(String[] parts, MysqlExecutionConfig config, final Map configMap) throws ConfigurationException { + public List transformSql(String[] parts, MysqlExecutionConfig config, final Map configMap) + throws ConfigurationException { + + logger.info("Config Map contains " + configMap.entrySet().stream().map(e -> e.getKey() + " : " + e.getValue()).collect(Collectors.joining(","))); + List result = new ArrayList<>(); String prospectiveSchema = config.getProspectiveVersion(); if (prospectiveSchema == null) { From 592eedeba9e8390132c36a0ff359aec7160cf2eb Mon Sep 17 00:00:00 2001 From: lukeswindale Date: Mon, 9 Oct 2023 20:35:01 +1000 Subject: [PATCH 08/11] Fixing up the config properties in the code merged from SI --- .../ihtsdo/rvf/core/service/AssertionExecutionService.java | 2 +- .../rvf/core/service/util/MySqlQueryTransformer.java | 7 ++----- .../rvf/core/service/util/MySqlQueryTransformerTest.java | 5 +++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java index 209671416..1ec7d8df9 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/AssertionExecutionService.java @@ -226,7 +226,7 @@ private String[] splitCommand(ExecutionCommand command) { private List transformSql(String[] parts, Assertion assertion, MysqlExecutionConfig config) throws ConfigurationException { String qaResult = dataSource.getDefaultCatalog()+ "." + qaResulTableName; MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); - Map configMap = Map.of("qa_result",qaResult, "", String.valueOf(assertion.getAssertionId())); + Map configMap = Map.of("qa_result", qaResult, "", String.valueOf(assertion.getAssertionId())); return queryTransformer.transformSql(parts, config, configMap); } diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index aa4d4bde0..f73e420e8 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -67,10 +67,7 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi part = part.replaceAll("", moduleId); part = part.replaceAll("", version); // watch out for any 's that users might have introduced - part = part.replaceAll("qa_result", - configMap.get("qa_result").startsWith(configMap.get("default_catalog")) - ? configMap.get("default_catalog") + "." + configMap.get("qa_result") - : configMap.get("qa_result")); + part = part.replaceAll("qa_result", configMap.get("qa_result")); part = part.replaceAll("", prospectiveSchema); part = part.replaceAll("", prospectiveSchema); if (previousReleaseSchema != null) { @@ -85,7 +82,7 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi part = part.replaceAll(Pattern.quote("[[:<:]]"),"\\\\b" ); part = part.replaceAll(Pattern.quote("[[:>:]]"),"\\\\b" ); for(Map.Entry configMapEntry: configMap.entrySet()){ - if (!configMapEntry.getKey().equals("qa_result")) { + if (configMapEntry.getKey().matches("^<[^>]+>$")) { part = part.replaceAll(configMapEntry.getKey(), configMapEntry.getValue()); } } diff --git a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java index e04dea664..a4351c9e8 100644 --- a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java +++ b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java @@ -139,7 +139,8 @@ public void transformSql() throws ConfigurationException { String testQaResult = "abc"; when(config.getProspectiveVersion()).thenReturn("rvf_au_20221231_230619110027"); when(config.getPreviousVersion()).thenReturn(String.valueOf(10)); - Map configMap = Map.of("qa_result",testQaResult, "", String.valueOf(testAssertionId)); + Map configMap = Map.of("qa_result",testQaResult, + "", String.valueOf(testAssertionId)); List result = queryTransformer.transformSql(sqlParts,config, configMap); assertEquals(1,result.size()); assertFalse(result.get(0).contains("")); @@ -164,4 +165,4 @@ public void transformSqlMissingConfig() { } -} \ No newline at end of file +} From a9aea957cbe7397d711f34bff8d3a96a1d68bd87 Mon Sep 17 00:00:00 2001 From: Senjo Jose Date: Tue, 20 Jun 2023 13:13:47 +1000 Subject: [PATCH 09/11] Fix regex compatibility --- .../rvf/core/service/util/MySqlQueryTransformerTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java index a4351c9e8..0db9ecf96 100644 --- a/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java +++ b/src/test/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformerTest.java @@ -132,7 +132,7 @@ public void transformSql() throws ConfigurationException { "\tunion\n" + " select referencedcomponentid, from .simplerefset_\n" + "\tunion\n" + - " select referencedcomponentid from .simplemaprefset_\n" + + " select referencedcomponentid from .simplemaprefset_ and RLIKE Binary '[[:<:]]hexachlorophene[[:>:]]'\n" + " )"; String[] sqlParts = {sql}; String testAssertionId = "xyz"; @@ -143,12 +143,14 @@ public void transformSql() throws ConfigurationException { "", String.valueOf(testAssertionId)); List result = queryTransformer.transformSql(sqlParts,config, configMap); assertEquals(1,result.size()); - assertFalse(result.get(0).contains("")); - assertFalse(result.get(0).contains("")); + assertTrue(!result.get(0).contains("")); + assertTrue(!result.get(0).contains("")); assertTrue(result.get(0).contains("rvf_au_20221231_230619110027.description_d")); + assertTrue(result.get(0).contains("'\\bhexachlorophene\\b'")); assertTrue(result.get(0).contains(testAssertionId)); } + @Test public void transformSqlMissingConfig() { MySqlQueryTransformer queryTransformer = new MySqlQueryTransformer(); From 7d918db4292b76aaae1178407c8991f3048b8843 Mon Sep 17 00:00:00 2001 From: lukeswindale Date: Mon, 12 Aug 2024 13:21:10 +1000 Subject: [PATCH 10/11] Resolving bad merge --- .../importer/AssertionsDatabaseImporter.java | 107 ++---------------- 1 file changed, 11 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java index 423d69e53..1c0a08b5f 100644 --- a/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java +++ b/src/main/java/org/ihtsdo/rvf/importer/AssertionsDatabaseImporter.java @@ -30,7 +30,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -/** /** * An implementation of a {@link org.ihtsdo.rvf.importer.AssertionsDatabaseImporter} that imports older * Release Assertion Toolkit content via XML and SQL files. The XML file defines assertions and SQL files are @@ -57,99 +56,6 @@ public boolean isAssertionImportRequired() { return assertions == null || assertions.isEmpty(); } - public List getAssertionsFromFile(final InputStream manifestInputStream) { - final Document xmlDocument = getJDomDocumentFromFile(manifestInputStream); - List assertions = new ArrayList<>(); - if(xmlDocument != null) - { - final XPathFactory factory = XPathFactory.instance(); - final XPathExpression expression = factory.compile("//script"); - final List scriptElements = expression.evaluate(xmlDocument); - if(scriptElements.size() > 0) { - // get various values from script element - for (final Element element : scriptElements) { - assertions.add(createAssertionFromElement(element)); - - } - } - } - return assertions; - } - - public void importAssertionsFromFile(final InputStream manifestInputStream, final String sqlResourcesFolderLocation){ - // get JDOM document from given manifest file - final Document xmlDocument = getJDomDocumentFromFile(manifestInputStream); - if(xmlDocument != null) - { - final XPathFactory factory = XPathFactory.instance(); - final XPathExpression expression = factory.compile("//script"); - final List scriptElements = expression.evaluate(xmlDocument); - if(scriptElements.size() > 0) - { - // get various values from script element - for(final Element element : scriptElements) - { - final Assertion assertion = createAssertionFromElement(element); - if(assertion != null){ - try - { - logger.info("Created assertion id : " + assertion.getAssertionId()); - assert UUID.fromString(element.getAttributeValue("uuid")).equals(assertion.getUuid()); - // get Sql file name from element and use it to add SQL test - final String sqlFileName = element.getAttributeValue("sqlFile"); - logger.info("sqlFileName = " + sqlFileName); - String category = element.getAttributeValue("category"); - /* - We know category is written as file-centric-validation, component-centric-validation, etc - We use this to generate the corresponding using folder name = category - validation - */ - final int index = category.indexOf("validation"); - if(index > -1) - { - category = category.substring(0, index-1); - } - logger.info("category = " + category); - - String sqlResourceFileName = sqlResourcesFolderLocation + RESOURCE_PATH_SEPARATOR + category + RESOURCE_PATH_SEPARATOR + sqlFileName; - InputStream sqlInputStream = null; - sqlInputStream = assertionResourceManager.readResourceStream(sqlResourceFileName); - if (sqlInputStream != null) { - final String sqlString = readStream(sqlInputStream); - // add test to assertion - if(isPreRequisitesSql(sqlFileName)){ //handle pre-requisites.sql - addPreRequisiteSqlToAssertion(assertion, sqlString); - } else{ - addSqlTestToAssertion(assertion, sqlString); - } - } else { - String msg = "Failed to find sql file name from source:" + sqlResourceFileName + " for assertion uuid:" + assertion.getUuid(); - logger.error(msg); - throw new IllegalStateException(msg); - } - } - catch (final Exception e) { - logger.warn("Error reading sql from input stream. Nested exception is : " + e.getMessage()); - throw new IllegalStateException("Failed to add sql script test to assertion with uuid:" + assertion.getUuid(), e); - } - } - else{ - throw new IllegalStateException("Error creating assertion"); - } - } - - // finally print all lookup map contents for debugging - //todo save somewhere? - logger.debug("lookupMap = " + lookupMap); - } - else{ - logger.error("There are no script elements to import in the XML file provided. Please note that the " + - "XML file should contain element named script"); - } - } - else{ - logger.warn("Error generating document from xml file passed : " + manifestInputStream); - } - } - /** * Imports assertions from the given manifest file input stream. The manifest file should be an XML file * containing a list of assertions to import. Each assertion should contain a list of SQL scripts to execute @@ -222,8 +128,18 @@ private void addSqlTestsToAssertion(Assertion assertion, Element element, String } final String sqlString = IOUtils.toString(sqlInputStream, UTF_8); // add test to assertion + if(isPreRequisitesSql(sqlFileName)){ //handle pre-requisites.sql + try { + addPreRequisiteSqlToAssertion(assertion, sqlString); + } catch (Exception e) { + String errorMsg = "Failed to add pre-requisite sql script test to assertion with uuid:" + assertion.getUuid(); + logger.error(errorMsg, e); + throw new IllegalStateException(errorMsg, e); + } + } else{ addSqlTestToAssertion(assertion, sqlString); } + } private boolean isPreRequisitesSql(String sqlFileName){ return sqlFileName.equalsIgnoreCase("pre-requisites.sql"); @@ -297,8 +213,7 @@ public void addSqlTestToAssertion(final Assertion assertion, final String sql){ uploadTest(assertion, sql, statements); } - protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) - throws RuntimeException { + protected void addPreRequisiteSqlToAssertion(final Assertion assertion, String preRequisiteSql) throws BusinessServiceException { MySqlQueryTransformer mySqlQueryTransformer = new MySqlQueryTransformer(); final List sqlStatements = mySqlQueryTransformer.transformToStatements(preRequisiteSql); uploadTest(assertion, preRequisiteSql, sqlStatements); From f33387da452d316102a45bea642e3f7964c94383 Mon Sep 17 00:00:00 2001 From: lukeswindale Date: Mon, 12 Aug 2024 13:27:04 +1000 Subject: [PATCH 11/11] Resolving bad merge --- .../org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java index f73e420e8..0ae5c0985 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/util/MySqlQueryTransformer.java @@ -48,7 +48,6 @@ public List transformSql(String[] parts, MysqlExecutionConfig config, fi throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema); } - final String[] nameParts = config.getProspectiveVersion().split("_"); String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED"); String includedModules = config.getIncludedModules().stream().collect(Collectors.joining(",")); String defaultModuleId = StringUtils.hasLength(config.getDefaultModuleId()) ? config.getDefaultModuleId() : (nameParts.length >= 2 ? AssertionGroupImporter.ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED");