diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java index ba9abe77ce0..5bae99d8832 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java @@ -62,6 +62,7 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.ScanUtil; +import org.apache.phoenix.util.SchemaUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,8 +93,9 @@ private class TransformingDataTableQueryPlanBuilder implements QueryPlanBuilder public QueryPlan getQueryPlan(PhoenixConnection phoenixConnection, String oldTableFullName, String newTableFullName) throws SQLException { PTable newTable = phoenixConnection.getTableNoCache(newTableFullName); + String quotedOldTableName = getQuotedTableNameForSQL(oldTableFullName); ServerBuildTransformingTableCompiler compiler = - new ServerBuildTransformingTableCompiler(phoenixConnection, oldTableFullName); + new ServerBuildTransformingTableCompiler(phoenixConnection, quotedOldTableName); MutationPlan plan = compiler.compile(newTable); return plan.getQueryPlan(); } @@ -104,8 +106,9 @@ private class DataTableQueryPlanBuilder implements QueryPlanBuilder { public QueryPlan getQueryPlan(PhoenixConnection phoenixConnection, String dataTableFullName, String indexTableFullName) throws SQLException { PTable indexTable = phoenixConnection.getTableNoCache(indexTableFullName); + String quotedDataTableName = getQuotedTableNameForSQL(dataTableFullName); ServerBuildIndexCompiler compiler = - new ServerBuildIndexCompiler(phoenixConnection, dataTableFullName); + new ServerBuildIndexCompiler(phoenixConnection, quotedDataTableName); MutationPlan plan = compiler.compile(indexTable); return plan.getQueryPlan(); } @@ -117,7 +120,8 @@ public QueryPlan getQueryPlan(PhoenixConnection phoenixConnection, String dataTa String indexTableFullName) throws SQLException { QueryPlan plan; try (final PhoenixStatement statement = new PhoenixStatement(phoenixConnection)) { - String query = "SELECT count(*) FROM " + indexTableFullName; + String quotedIndexName = getQuotedTableNameForSQL(indexTableFullName); + String query = "SELECT count(*) FROM " + quotedIndexName; plan = statement.compileQuery(query); TableRef tableRef = plan.getTableRef(); Scan scan = plan.getContext().getScan(); @@ -139,6 +143,12 @@ public QueryPlan getQueryPlan(PhoenixConnection phoenixConnection, String dataTa private QueryPlanBuilder queryPlanBuilder; + private String getQuotedTableNameForSQL(String normalizedFullTableName) { + String schemaName = SchemaUtil.getSchemaNameFromFullName(normalizedFullTableName); + String tableName = SchemaUtil.getTableNameFromFullName(normalizedFullTableName); + return SchemaUtil.getFullTableNameWithQuotes(schemaName, tableName); + } + @Override protected QueryPlan getQueryPlan(final JobContext context, final Configuration configuration) throws IOException { diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index 92d4d01c559..ada6604dce0 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -754,7 +754,7 @@ private Job configureJobForServerBuildIndex() throws Exception { configuration.setBooleanIfUnset( PhoenixConfigurationUtil.MAPREDUCE_RANDOMIZE_MAPPER_EXECUTION_ORDER, true); - PhoenixConfigurationUtil.setIndexToolDataTableName(configuration, dataTableWithSchema); + PhoenixConfigurationUtil.setIndexToolDataTableName(configuration, qDataTable); PhoenixConfigurationUtil.setIndexToolIndexTableName(configuration, qIndexTable); PhoenixConfigurationUtil.setIndexToolSourceTable(configuration, sourceTable); if (startTime != null) { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index 3d03378dbba..d5c12b0bbde 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -34,6 +34,8 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNKNOWN_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_UNVERIFIED_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REBUILD_VALID_INDEX_ROW_COUNT; +import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_UNVERIFIED_INDEX_ROW_COUNT; +import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEFORE_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; @@ -817,6 +819,86 @@ public void testCaseSensitiveNames() throws Exception { } } + @Test + public void testIndexToDataVerification() throws Exception { + testIndexToDataVerificationHelper(false); + } + + @Test + public void testIndexToDataVerificationCaseSensitive() throws Exception { + testIndexToDataVerificationHelper(true); + } + + private void testIndexToDataVerificationHelper(boolean caseSensitive) throws Exception { + if (localIndex || transactional || useSnapshot) { + return; + } + String schemaName = caseSensitive ? generateUniqueName().toLowerCase() : generateUniqueName(); + String dataTableName = + caseSensitive ? generateUniqueName().toLowerCase() : generateUniqueName(); + String indexTableName = + caseSensitive ? generateUniqueName().toLowerCase() : generateUniqueName(); + String sSchemaName = caseSensitive ? SchemaUtil.getEscapedArgument(schemaName) : schemaName; + String sDataTableName = + caseSensitive ? SchemaUtil.getEscapedArgument(dataTableName) : dataTableName; + String sIndexTableName = + caseSensitive ? SchemaUtil.getEscapedArgument(indexTableName) : indexTableName; + String qDataTableName = caseSensitive + ? SchemaUtil.getFullTableNameWithQuotes(schemaName, dataTableName) + : SchemaUtil.getTableName(schemaName, dataTableName); + String qIndexTableName = caseSensitive + ? SchemaUtil.getFullTableNameWithQuotes(schemaName, indexTableName) + : SchemaUtil.getTableName(schemaName, indexTableName); + + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + if (namespaceMapped) { + conn.createStatement().execute("CREATE SCHEMA " + sSchemaName); + } + conn.createStatement().execute("CREATE TABLE " + qDataTableName + + " (ID INTEGER NOT NULL PRIMARY KEY, VAL1 INTEGER, VAL2 INTEGER) " + tableDDLOptions); + + PreparedStatement dataStmt = + conn.prepareStatement("UPSERT INTO " + qDataTableName + " VALUES(?,?,?)"); + for (int i = 1; i <= 4; i++) { + dataStmt.setInt(1, i); + dataStmt.setInt(2, i + 1); + dataStmt.setInt(3, i * 2); + dataStmt.execute(); + } + conn.commit(); + + conn.createStatement() + .execute(String.format("CREATE INDEX %s ON %s (VAL1) INCLUDE (VAL2) " + indexDDLOptions, + sIndexTableName, qDataTableName)); + + // Add extra rows directly to the index table + PreparedStatement indexStmt = + conn.prepareStatement("UPSERT INTO " + qIndexTableName + " VALUES(?,?,?)"); + for (int i = 10; i <= 12; i++) { + indexStmt.setInt(1, i + 1); + indexStmt.setInt(2, i); + indexStmt.setInt(3, i * 2); + indexStmt.execute(); + } + conn.commit(); + + // Run index-to-data verification + IndexTool indexTool = runIndexTool(false, sSchemaName, sDataTableName, sIndexTableName, null, + 0, IndexTool.IndexVerifyType.ONLY, "-fi"); + + CounterGroup mrJobCounters = getMRJobCounters(indexTool); + // Extra index rows should be detected + if (mutable) { + assertEquals(3, mrJobCounters + .findCounter(BEFORE_REPAIR_EXTRA_UNVERIFIED_INDEX_ROW_COUNT.name()).getValue()); + } else { + assertEquals(3, mrJobCounters + .findCounter(BEFORE_REPAIR_EXTRA_VERIFIED_INDEX_ROW_COUNT.name()).getValue()); + } + } + } + public static void assertExplainPlan(boolean localIndex, String actualExplainPlan, String dataTableFullName, String indexTableFullName) { assertExplainPlan(localIndex, actualExplainPlan, dataTableFullName, indexTableFullName, true);