From e05714cd815ee3b0c1b2a7c0833815fab1fe2728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A5=A0?= Date: Thu, 29 Nov 2018 22:13:52 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=A4=9A=E7=BA=A7=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E6=9C=AA=E5=AE=8C=E6=88=90=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/loader/xml/XMLRuleLoader.java | 47 ++++--- .../config/loader/xml/XMLSchemaLoader.java | 2 +- .../entity/rule/tablerule/SubTableRule.java | 31 +++++ .../entity/rule/tablerule/TableRule.java | 29 +++-- .../io/mycat/config/model/SchemaConfig.java | 17 +++ .../io/mycat/config/model/TableConfig.java | 30 +++-- .../config/model/rule/TableRuleConfig.java | 21 +++- .../java/io/mycat/route/RouteResultset.java | 11 +- .../route/impl/DruidMycatRouteStrategy.java | 8 +- .../java/io/mycat/route/util/RouterUtil.java | 118 +++++++++++++++++- src/main/resources/rule.dtd | 5 +- .../io/mycat/route/TestSubTableRuleRoute.java | 69 ++++++++++ .../function/RuleFunctionSuitTableTest.java | 14 ++- src/test/resources/route/sub_tables/rule.xml | 33 +++++ .../resources/route/sub_tables/schema.xml | 26 ++++ src/test/resources/rule2.xml | 52 ++++++++ 16 files changed, 462 insertions(+), 51 deletions(-) create mode 100644 src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/SubTableRule.java create mode 100644 src/test/java/io/mycat/route/TestSubTableRuleRoute.java create mode 100644 src/test/resources/route/sub_tables/rule.xml create mode 100644 src/test/resources/route/sub_tables/schema.xml create mode 100644 src/test/resources/rule2.xml diff --git a/src/main/java/io/mycat/config/loader/xml/XMLRuleLoader.java b/src/main/java/io/mycat/config/loader/xml/XMLRuleLoader.java index 084fcff1d..087fece28 100644 --- a/src/main/java/io/mycat/config/loader/xml/XMLRuleLoader.java +++ b/src/main/java/io/mycat/config/loader/xml/XMLRuleLoader.java @@ -133,30 +133,39 @@ private void loadTableRules(Element root) throws SQLSyntaxErrorException { throw new ConfigException("table rule " + name + " duplicated!"); } - //获取rule标签 - NodeList ruleNodes = e.getElementsByTagName("rule"); - int length = ruleNodes.getLength(); - if (length > 1) { - throw new ConfigException("only one rule can defined :" - + name); - } - //目前只处理第一个，未来可能有多列复合逻辑需求 - //RuleConfig是保存着rule与function对应关系的对象 - RuleConfig rule = loadRule((Element) ruleNodes.item(0)); - String funName = rule.getFunctionName(); - //判断function是否存在，获取function - AbstractPartitionAlgorithm func = functions.get(funName); - if (func == null) { - throw new ConfigException("can't find function of name :" - + funName); - } - rule.setRuleAlgorithm(func); + RuleConfig rule = loadRuleConfig(e,"rule"); + RuleConfig subTableRule = loadRuleConfig(e,"subTableRule"); //保存到tableRules - tableRules.put(name, new TableRuleConfig(name, rule)); + tableRules.put(name, new TableRuleConfig(name, rule,subTableRule)); } } } + private RuleConfig loadRuleConfig(Element e,String tagName) throws SQLSyntaxErrorException { + //获取rule标签 + NodeList ruleNodes = e.getElementsByTagName(tagName); + int length = ruleNodes.getLength(); + if(length == 0){ + return null; + } + if (length > 1) { + throw new ConfigException("only one "+tagName+" can defined :" + + e.getAttribute("name")); + } + //目前只处理第一个，未来可能有多列复合逻辑需求 + //RuleConfig是保存着rule与function对应关系的对象 + RuleConfig rule = loadRule((Element) ruleNodes.item(0)); + String funName = rule.getFunctionName(); + //判断function是否存在，获取function + AbstractPartitionAlgorithm func = functions.get(funName); + if (func == null) { + throw new ConfigException("can't find function of name :" + + funName); + } + rule.setRuleAlgorithm(func); + return rule; + } + private RuleConfig loadRule(Element element) throws SQLSyntaxErrorException { //读取columns Element columnsEle = ConfigUtil.loadElement(element, "columns"); diff --git a/src/main/java/io/mycat/config/loader/xml/XMLSchemaLoader.java b/src/main/java/io/mycat/config/loader/xml/XMLSchemaLoader.java index 65fda5329..6cfb09dcb 100644 --- a/src/main/java/io/mycat/config/loader/xml/XMLSchemaLoader.java +++ b/src/main/java/io/mycat/config/loader/xml/XMLSchemaLoader.java @@ -387,7 +387,7 @@ private Map loadTables(Element node) { TableConfig table = new TableConfig(tableName, primaryKey, autoIncrement, needAddLimit, tableType, dataNode, getDbType(dataNode), - (tableRuleConfig != null) ? tableRuleConfig.getRule() : null, + tableRuleConfig , ruleRequired, null, false, null, null,subTables); checkDataNodeExists(table.getDataNodes()); diff --git a/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/SubTableRule.java b/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/SubTableRule.java new file mode 100644 index 000000000..4ccde5d6d --- /dev/null +++ b/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/SubTableRule.java @@ -0,0 +1,31 @@ +package io.mycat.config.loader.zkprocess.entity.rule.tablerule; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlType; + +/** + * @author liunan by 2018/8/29 + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "subTableRule", propOrder = { "columns", "algorithm" }) +public class SubTableRule { + protected String columns; + protected String algorithm; + + public String getColumns() { + return columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + + public String getAlgorithm() { + return algorithm; + } + + public void setAlgorithm(String algorithm) { + this.algorithm = algorithm; + } +} diff --git a/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/TableRule.java b/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/TableRule.java index 1a1286d89..c34d9b301 100644 --- a/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/TableRule.java +++ b/src/main/java/io/mycat/config/loader/zkprocess/entity/rule/tablerule/TableRule.java @@ -30,10 +30,12 @@ public class TableRule implements Named { @XmlElement(required = true, name = "rule") protected Rule rule; + + @XmlElement(required = false, name = "subTableRule") + protected SubTableRule subTableRule; @XmlAttribute(required = true) protected String name; - public Rule getRule() { return rule; } @@ -52,15 +54,22 @@ public TableRule setName(String name) { return this; } + public SubTableRule getSubTableRule() { + return subTableRule; + } + + public void setSubTableRule( + SubTableRule subTableRule) { + this.subTableRule = subTableRule; + } + @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("TableRule [rule="); - builder.append(rule); - builder.append(", name="); - builder.append(name); - builder.append("]"); - return builder.toString(); + public String toString() { + final StringBuffer sb = new StringBuffer("TableRule{"); + sb.append("rule=").append(rule); + sb.append(", subTableRule=").append(subTableRule); + sb.append(", name='").append(name).append('\''); + sb.append('}'); + return sb.toString(); } - } diff --git a/src/main/java/io/mycat/config/model/SchemaConfig.java b/src/main/java/io/mycat/config/model/SchemaConfig.java index 2002590b4..2970a500c 100644 --- a/src/main/java/io/mycat/config/model/SchemaConfig.java +++ b/src/main/java/io/mycat/config/model/SchemaConfig.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Random; import java.util.Set; @@ -56,6 +57,7 @@ public class SchemaConfig { private final String[] allDataNodeStrArr; private Map dataNodeDbTypeMap=new HashMap<>(); + private boolean isMutilRoute ; public SchemaConfig(String name, String dataNode, Map tables, int defaultMaxLimit, @@ -81,8 +83,20 @@ public SchemaConfig(String name, String dataNode, } else { this.allDataNodeStrArr = null; } + + isMutilRoute = this.hasMutilRoute(); } + //table中有一个存在mutilRoute就算有多级路由 + private boolean hasMutilRoute(){ + Iterator it = this.tables.values().iterator(); + while (it.hasNext()){ + if(it.next().isMutilRoute()){ + return true; + } + } + return false; + } public String getDefaultDataNodeDbType() { return defaultDataNodeDbType; @@ -215,4 +229,7 @@ private static boolean isEmpty(String str) { return ((str == null) || (str.length() == 0)); } + public boolean isMutilRoute() { + return isMutilRoute; + } } \ No newline at end of file diff --git a/src/main/java/io/mycat/config/model/TableConfig.java b/src/main/java/io/mycat/config/model/TableConfig.java index dd2ede1a3..4d655f097 100644 --- a/src/main/java/io/mycat/config/model/TableConfig.java +++ b/src/main/java/io/mycat/config/model/TableConfig.java @@ -24,10 +24,8 @@ package io.mycat.config.model; import java.util.*; -import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.statement.SQLTableElement; import io.mycat.config.model.rule.RuleConfig; import io.mycat.util.SplitUtil; @@ -63,10 +61,13 @@ public class TableConfig { private volatile String tableStructureSQL; private volatile Map> dataNodeTableStructureSQLMap; private ReentrantReadWriteLock reentrantReadWriteLock = new ReentrantReadWriteLock(false); - + //mutileRoute + private final boolean isMutilRoute; + private final RuleConfig subTableRule; + private final String subPartitionColumn; public TableConfig(String name, String primaryKey, boolean autoIncrement,boolean needAddLimit, int tableType, - String dataNode,Set dbType, RuleConfig rule, boolean ruleRequired, + String dataNode,Set dbType, io.mycat.config.model.rule.TableRuleConfig tableRuleConfig, boolean ruleRequired, TableConfig parentTC, boolean isChildTable, String joinKey, String parentKey,String subTables) { if (name == null) { @@ -79,7 +80,7 @@ public TableConfig(String name, String primaryKey, boolean autoIncrement,boolean this.needAddLimit=needAddLimit; this.tableType = tableType; this.dbTypes=dbType; - if (ruleRequired && rule == null) { + if (ruleRequired && tableRuleConfig == null) { throw new IllegalArgumentException("ruleRequired but rule is null"); } @@ -107,8 +108,11 @@ public TableConfig(String name, String primaryKey, boolean autoIncrement,boolean this.distTables = new ArrayList(); } - this.rule = rule; + this.rule = tableRuleConfig.getRule(); + this.subTableRule = tableRuleConfig.getSubTableRule(); + this.isMutilRoute = subTableRule != null; //有subTableRule时,认是多级路由 this.partitionColumn = (rule == null) ? null : rule.getColumn(); + this.subPartitionColumn = isMutilRoute ? subTableRule.getColumn() : null ; partionKeyIsPrimaryKey=(partitionColumn==null)?primaryKey==null:partitionColumn.equals(primaryKey); this.ruleRequired = ruleRequired; this.childTable = isChildTable; @@ -264,7 +268,7 @@ public ArrayList getDistTables() { } public boolean isDistTable(){ - if(this.distTables!=null && !this.distTables.isEmpty() ){ + if(this.distTables!=null && !this.distTables.isEmpty() && !isMutilRoute){ return true; } return false; @@ -302,4 +306,16 @@ public Map> getDataNodeTableStructureSQLMap() { public void setDataNodeTableStructureSQLMap(Map> dataNodeTableStructureSQLMap) { this.dataNodeTableStructureSQLMap = dataNodeTableStructureSQLMap; } + + public boolean isMutilRoute() { + return isMutilRoute; + } + + public RuleConfig getSubTableRule() { + return subTableRule; + } + + public String getSubPartitionColumn() { + return subPartitionColumn; + } } \ No newline at end of file diff --git a/src/main/java/io/mycat/config/model/rule/TableRuleConfig.java b/src/main/java/io/mycat/config/model/rule/TableRuleConfig.java index 8f44be2b2..8dff40e8e 100644 --- a/src/main/java/io/mycat/config/model/rule/TableRuleConfig.java +++ b/src/main/java/io/mycat/config/model/rule/TableRuleConfig.java @@ -32,7 +32,22 @@ public class TableRuleConfig implements Serializable { private String name; private final RuleConfig rule; - public TableRuleConfig(String name, RuleConfig rule) { + private final RuleConfig subTableRule; + +// public TableRuleConfig(String name, RuleConfig rule) { +// if (name == null) { +// throw new IllegalArgumentException("name is null"); +// } +// this.name = name; +// if (rule == null) { +// throw new IllegalArgumentException("no rule is found"); +// } +// this.rule =rule; +// this.subTableRule = null; +// } + + public TableRuleConfig(String name, RuleConfig rule, + RuleConfig subTableRule) { if (name == null) { throw new IllegalArgumentException("name is null"); } @@ -41,6 +56,7 @@ public TableRuleConfig(String name, RuleConfig rule) { throw new IllegalArgumentException("no rule is found"); } this.rule =rule; + this.subTableRule = subTableRule; } public String getName() { @@ -58,4 +74,7 @@ public RuleConfig getRule() { return rule; } + public RuleConfig getSubTableRule() { + return subTableRule; + } } diff --git a/src/main/java/io/mycat/route/RouteResultset.java b/src/main/java/io/mycat/route/RouteResultset.java index 9b3b0a182..db556ae1e 100644 --- a/src/main/java/io/mycat/route/RouteResultset.java +++ b/src/main/java/io/mycat/route/RouteResultset.java @@ -80,6 +80,7 @@ public final class RouteResultset implements Serializable { private boolean selectForUpdate; + private boolean isMutilRoute; public boolean isSelectForUpdate() { return selectForUpdate; } @@ -426,7 +427,15 @@ public boolean isDistTable(){ return false; } - @Override + public boolean isMutilRoute() { + return isMutilRoute; + } + + public void setMutilRoute(boolean mutilRoute) { + isMutilRoute = mutilRoute; + } + + @Override public String toString() { StringBuilder s = new StringBuilder(); s.append(statement).append(", route={"); diff --git a/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java b/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java index ba33f1e0c..5dbcc2cfa 100644 --- a/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java +++ b/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java @@ -233,7 +233,6 @@ public RouteResultset routeNormalSqlWithAST(SchemaConfig schema, /** * 子查询中存在关联查询的情况下,检查关联字段是否是分片字段 * @param rulemap - * @param ships * @return */ private boolean checkRuleField(Map rulemap,MycatSchemaStatVisitor visitor){ @@ -439,7 +438,12 @@ private RouteResultset directRoute(RouteResultset rrs,DruidShardingParseInfo ctx * subTables="t_order$1-2,t_order3" *目前分表 1.6 开始支持 幵丏 dataNode 在分表条件下只能配置一个，分表条件下不支持join。 */ - if(rrs.isDistTable()){ + if(rrs.isDistTable() && !schema.isMutilRoute()){ + return this.routeDisTable(statement,rrs); + } + + + if(schema.isMutilRoute()){ return this.routeDisTable(statement,rrs); } return rrs; diff --git a/src/main/java/io/mycat/route/util/RouterUtil.java b/src/main/java/io/mycat/route/util/RouterUtil.java index b5bb73cb4..7afce6b4d 100644 --- a/src/main/java/io/mycat/route/util/RouterUtil.java +++ b/src/main/java/io/mycat/route/util/RouterUtil.java @@ -1097,7 +1097,9 @@ public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardin routeToDistTableNode(tableName,schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect); return rrs; } - + if(tableConfig.isMutilRoute()){ + routeToMutilDistTableNode(tableName,schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect); + } if(retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) { // mulit routes ,not cache route result if (isSelect) { @@ -1140,7 +1142,6 @@ public static RouteResultset tryRouteForOneTable(SchemaConfig schema, DruidShard if(tc.isDistTable()){ return routeToDistTableNode(tableName,schema,rrs,ctx.getSql(), routeUnit.getTablesAndConditions(), cachePool,isSelect); } - if(tc.isGlobalTable()) {//全局表 if(isSelect) { // global select ,not cache route result @@ -1284,6 +1285,114 @@ private static RouteResultset routeToDistTableNode(String tableName, SchemaConfi return rrs; } + private static RouteResultset routeToMutilDistTableNode(String tableName, SchemaConfig schema, RouteResultset rrs, + String orgSql, Map>> tablesAndConditions, + LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException { + + TableConfig tableConfig = schema.getTables().get(tableName); + if(tableConfig == null) { + String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName(); + LOGGER.warn(msg); + throw new SQLNonTransientException(msg); + } + if(tableConfig.isGlobalTable()){ + String msg = "can't suport district table " + tableName + " schema:" + schema.getName() + " for global table "; + LOGGER.warn(msg); + throw new SQLNonTransientException(msg); + } + String partionCol = tableConfig.getSubPartitionColumn(); +// String primaryKey = tableConfig.getPrimaryKey(); + boolean isLoadData=false; + + Set tablesRouteSet = new HashSet(); + + List dataNodes = tableConfig.getDataNodes(); + //所有节点的分表逻辑是一样的,所以就取一个就可以了 + String dataNode = dataNodes.get(0); + + //主键查找缓存暂时不实现 + if(tablesAndConditions.isEmpty()){ + List subTables = tableConfig.getDistTables(); + tablesRouteSet.addAll(subTables); + } + + for(Map.Entry>> entry : tablesAndConditions.entrySet()) { + boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null; + Map> columnsMap = entry.getValue(); + + Set partitionValue = columnsMap.get(partionCol); + if(partitionValue == null || partitionValue.size() == 0) { + tablesRouteSet.addAll(tableConfig.getDistTables()); + } else { + for(ColumnRoutePair pair : partitionValue) { + AbstractPartitionAlgorithm algorithm = tableConfig.getSubTableRule().getRuleAlgorithm(); + if(pair.colValue != null) { + Integer tableIndex = algorithm.calculate(pair.colValue); + if(tableIndex == null) { + String msg = "can't find any valid datanode :" + tableConfig.getName() + + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue; + LOGGER.warn(msg); + throw new SQLNonTransientException(msg); + } + String subTable = tableConfig.getDistTables().get(tableIndex); + if(subTable != null) { + tablesRouteSet.add(subTable); + if(algorithm instanceof SlotFunction){ + rrs.getDataNodeSlotMap().put(subTable,((SlotFunction) algorithm).slotValue()); + } + } + } + if(pair.rangeValue != null) { + Integer[] tableIndexs = algorithm + .calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString()); + for(Integer idx : tableIndexs) { + String subTable = tableConfig.getDistTables().get(idx); + if(subTable != null) { + tablesRouteSet.add(subTable); + if(algorithm instanceof SlotFunction){ + rrs.getDataNodeSlotMap().put(subTable,((SlotFunction) algorithm).slotValue()); + } + } + } + } + } + } + } + + Object[] subTables = tablesRouteSet.toArray(); + //所有的dn都改写sql + RouteResultsetNode[] nodes = new RouteResultsetNode[subTables.length * dataNodes.size()]; + Map dataNodeSlotMap= rrs.getDataNodeSlotMap(); + for(int i=0;i0){ + if(tableConfig.isDistTable()){ routeToDistTableNode(tableName,schema,rrs,sql, tablesAndConditions, cachePool,isSelect); } + if(tableConfig.isMutilRoute()){ + routeToMutilDistTableNode(tableName,schema,rrs,sql, tablesAndConditions, cachePool,isSelect); + } //全局表或者不分库的表略过（全局表后面再计算） if(tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) { continue; diff --git a/src/main/resources/rule.dtd b/src/main/resources/rule.dtd index 303acef9f..9395d9e8a 100644 --- a/src/main/resources/rule.dtd +++ b/src/main/resources/rule.dtd @@ -16,13 +16,16 @@ - + + + + diff --git a/src/test/java/io/mycat/route/TestSubTableRuleRoute.java b/src/test/java/io/mycat/route/TestSubTableRuleRoute.java new file mode 100644 index 000000000..a034a8d17 --- /dev/null +++ b/src/test/java/io/mycat/route/TestSubTableRuleRoute.java @@ -0,0 +1,69 @@ +package io.mycat.route; + +import io.mycat.MycatServer; +import io.mycat.SimpleCachePool; +import io.mycat.cache.LayerCachePool; +import io.mycat.config.loader.SchemaLoader; +import io.mycat.config.loader.xml.XMLRuleLoader; +import io.mycat.config.loader.xml.XMLSchemaLoader; +import io.mycat.config.model.SchemaConfig; +import io.mycat.config.model.SystemConfig; +import io.mycat.route.factory.RouteStrategyFactory; +import java.sql.SQLNonTransientException; +import java.util.Map; +import junit.framework.Assert; +import org.junit.Test; + +/** + * @author liunan by 2018/8/29 + */ +public class TestSubTableRuleRoute { + + protected Map schemaMap; + protected LayerCachePool cachePool = new SimpleCachePool(); + + public TestSubTableRuleRoute() { + String schemaFile = "/route/sub_tables/schema.xml"; + String ruleFile = "/route/sub_tables/rule.xml"; + SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); + schemaMap = schemaLoader.getSchemas(); + MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); + RouteStrategyFactory.init(); + } + + + @Test + public void testSelect() throws SQLNonTransientException { + String sql = "select * from offer_detail where offer_id between 1 and 33"; + SchemaConfig schema = schemaMap.get("cndb"); + RouteResultset rrs = RouteStrategyFactory.getRouteStrategy().route(new SystemConfig(),schema, -1, sql, null, + null, cachePool); + Assert.assertEquals(5, rrs.getNodes().length); + } + + @Test + public void testInsert() throws SQLNonTransientException { + String sql = "insert into sqtestmonth (id,name,create_time) values(1,'sq1', '2017-5-12')"; + SchemaConfig schema = schemaMap.get("cndb"); + RouteResultset rrs = RouteStrategyFactory + .getRouteStrategy() + .route(new SystemConfig(),schema, -1, sql, null, + null, cachePool); + System.out.println(rrs.getNodes()[0]); + Assert.assertTrue(rrs.getNodes()[0].getStatement().contains("sqtestmonth20175")); + Assert.assertEquals(2, rrs.getNodes().length); + } + + @Test + public void testSelect2() throws SQLNonTransientException { + String sql = "select * from sqtestmonth where id = 1 and create_time = '2017-5-12'"; + SchemaConfig schema = schemaMap.get("cndb"); + RouteResultset rrs = RouteStrategyFactory + .getRouteStrategy() + .route(new SystemConfig(),schema, -1, sql, null, + null, cachePool); + System.out.println(rrs.getNodes()[0]); + Assert.assertTrue(rrs.getNodes()[0].getStatement().contains("sqtestmonth20175")); + Assert.assertEquals(2, rrs.getNodes().length); + } +} diff --git a/src/test/java/io/mycat/route/function/RuleFunctionSuitTableTest.java b/src/test/java/io/mycat/route/function/RuleFunctionSuitTableTest.java index 80667c3f5..574fc835b 100644 --- a/src/test/java/io/mycat/route/function/RuleFunctionSuitTableTest.java +++ b/src/test/java/io/mycat/route/function/RuleFunctionSuitTableTest.java @@ -1,5 +1,6 @@ package io.mycat.route.function; +import io.mycat.config.model.rule.TableRuleConfig; import java.util.Arrays; import org.junit.Assert; @@ -25,8 +26,9 @@ public void testAutoPartitionByLong() { Assert.assertEquals(3, autoPartition.getPartitionNum()); RuleConfig rule = new RuleConfig("id", "auto-partition-long"); rule.setRuleAlgorithm(autoPartition); + TableRuleConfig tableRuleConfig = new TableRuleConfig("tb",rule,null); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2", - null, rule, true, null, false, null, null, null); + null, tableRuleConfig, true, null, false, null, null, null); int suit1 = autoPartition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); @@ -55,7 +57,7 @@ public void testAutoPartitionByLong() { RuleConfig rule2 = new RuleConfig("id", "auto-partition-long-dupl"); rule2.setRuleAlgorithm(autoPartition2); TableConfig tableConf2 = new TableConfig("test2", "id", true, false, -1, "dn1,dn2", - null, rule, true, null, false, null, null, null); + null, new TableRuleConfig("tb",rule,null), true, null, false, null, null, null); Assert.assertEquals(0, autoPartition2.suitableFor(tableConf2)); Assert.assertEquals(0, autoPartition2.calculate("500").intValue()); @@ -79,7 +81,7 @@ public void testPartitionByDate() { RuleConfig rule = new RuleConfig("col_date", "partition-date"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2,dn3", - null, rule, true, null, false, null, null, null); + null, new TableRuleConfig("tb",rule,null), true, null, false, null, null, null); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); @@ -115,7 +117,7 @@ public void testPartitionByHashMod() { RuleConfig rule = new RuleConfig("id", "partition-hash-mod"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2,dn3", - null, rule, true, null, false, null, null, null); + null, new TableRuleConfig("tb",rule,null), true, null, false, null, null, null); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(0, suit1); @@ -140,7 +142,7 @@ public void testPartitionByRangeMod() { RuleConfig rule = new RuleConfig("id", "partition-range-mod"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn$1-10", - null, rule, true, null, false, null, null, null); + null, new TableRuleConfig("tb",rule,null), true, null, false, null, null, null); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); @@ -205,7 +207,7 @@ public void testPartitionByPrefixPattern() { RuleConfig rule = new RuleConfig("id", "partition-prefix-pattern"); rule.setRuleAlgorithm(partition); TableConfig tableConf = new TableConfig("test", "id", true, false, -1, "dn1,dn2", - null, rule, true, null, false, null, null, null); + null, new TableRuleConfig("tb",rule,null), true, null, false, null, null, null); int suit1 = partition.suitableFor(tableConf); Assert.assertEquals(-1, suit1); diff --git a/src/test/resources/route/sub_tables/rule.xml b/src/test/resources/route/sub_tables/rule.xml new file mode 100644 index 000000000..12d613937 --- /dev/null +++ b/src/test/resources/route/sub_tables/rule.xml @@ -0,0 +1,33 @@ + + + + + + + + + id + mod-long + + + create_time + partbymonth + + + + yyyy-MM-dd + 2017-01-01 + + + + 2 + + diff --git a/src/test/resources/route/sub_tables/schema.xml b/src/test/resources/route/sub_tables/schema.xml new file mode 100644 index 000000000..f51c76669 --- /dev/null +++ b/src/test/resources/route/sub_tables/schema.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + select user() + + + + diff --git a/src/test/resources/rule2.xml b/src/test/resources/rule2.xml new file mode 100644 index 000000000..2414b1c32 --- /dev/null +++ b/src/test/resources/rule2.xml @@ -0,0 +1,52 @@ + + + + + + + + + id + mod-long + + + + age + func1 + + + + + + + + id + func1 + + + + + + 2 + 512 + + + + + 2 + + From 8e132b32e9da08c4ebf705a2290368a6be1794a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A5=A0?= Date: Sun, 2 Dec 2018 13:32:57 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=A4=9A=E7=BA=A7=E8=B7=AF=E7=94=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/mycat/config/model/SchemaConfig.java | 3 + .../route/impl/DruidMycatRouteStrategy.java | 62 ++++++++++++++++--- .../parser/druid/impl/DruidInsertParser.java | 25 ++++++++ .../java/io/mycat/route/util/RouterUtil.java | 56 +++++++++++++---- .../handler/ServerLoadDataInfileHandler.java | 20 +++++- .../io/mycat/route/TestDisTableRuleRoute.java | 46 ++++++++++++++ .../io/mycat/route/TestSubTableRuleRoute.java | 22 +++++-- src/test/resources/route/disRoute/rule.xml | 29 +++++++++ src/test/resources/route/disRoute/schema.xml | 26 ++++++++ .../route/{sub_tables => mulitRoute}/rule.xml | 0 .../{sub_tables => mulitRoute}/schema.xml | 0 11 files changed, 264 insertions(+), 25 deletions(-) create mode 100644 src/test/java/io/mycat/route/TestDisTableRuleRoute.java create mode 100644 src/test/resources/route/disRoute/rule.xml create mode 100644 src/test/resources/route/disRoute/schema.xml rename src/test/resources/route/{sub_tables => mulitRoute}/rule.xml (100%) rename src/test/resources/route/{sub_tables => mulitRoute}/schema.xml (100%) diff --git a/src/main/java/io/mycat/config/model/SchemaConfig.java b/src/main/java/io/mycat/config/model/SchemaConfig.java index 2970a500c..9aae67945 100644 --- a/src/main/java/io/mycat/config/model/SchemaConfig.java +++ b/src/main/java/io/mycat/config/model/SchemaConfig.java @@ -57,6 +57,9 @@ public class SchemaConfig { private final String[] allDataNodeStrArr; private Map dataNodeDbTypeMap=new HashMap<>(); + /** + * 是否为多级路由 + */ private boolean isMutilRoute ; public SchemaConfig(String name, String dataNode, diff --git a/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java b/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java index 5dbcc2cfa..5b2c677f4 100644 --- a/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java +++ b/src/main/java/io/mycat/route/impl/DruidMycatRouteStrategy.java @@ -82,8 +82,8 @@ public class DruidMycatRouteStrategy extends AbstractRouteStrategy { middlerResultHandler.put(SQLExistsExpr.class, new SQLExistsResultHandler()); middlerResultHandler.put(SQLAllExpr.class, new SQLAllResultHandler()); } - - + + @Override public RouteResultset routeNormalSqlWithAST(SchemaConfig schema, String stmt, RouteResultset rrs,String charset, @@ -438,17 +438,65 @@ private RouteResultset directRoute(RouteResultset rrs,DruidShardingParseInfo ctx * subTables="t_order$1-2,t_order3" *目前分表 1.6 开始支持 幵丏 dataNode 在分表条件下只能配置一个，分表条件下不支持join。 */ - if(rrs.isDistTable() && !schema.isMutilRoute()){ +// if(rrs.isDistTable() && !schema.isMutilRoute()){ +// return this.routeDisTable(statement,rrs); +// } +// +// + if(schema.isMutilRoute()){ + return this.routeMutil(schema,statement,rrs); + } + if(rrs.isDistTable()){ return this.routeDisTable(statement,rrs); } + return rrs; + } - - if(schema.isMutilRoute()){ - return this.routeDisTable(statement,rrs); + /** + * 多级路由 + * @param schemaConfig + * @param statement + * @param rrs + * @return + */ + private RouteResultset routeMutil(SchemaConfig schemaConfig,SQLStatement statement, RouteResultset rrs) + throws SQLSyntaxErrorException { + if(!schemaConfig.isMutilRoute()){ + return rrs; + } + if(statement instanceof SQLInsertStatement) { + SQLInsertStatement insertStatement = (SQLInsertStatement) statement; + SQLExprTableSource tableSource = insertStatement.getTableSource(); + for (RouteResultsetNode node : rrs.getNodes()) { + SQLExprTableSource from2 = getDisTable(tableSource, node); + insertStatement.setTableSource(from2); + node.setStatement(insertStatement.toString()); + } } + SQLTableSource tableSource; + if(statement instanceof SQLDeleteStatement) { + SQLDeleteStatement deleteStatement = (SQLDeleteStatement) statement; + tableSource = deleteStatement.getTableSource(); + for (RouteResultsetNode node : rrs.getNodes()) { + SQLExprTableSource from2 = getDisTable(tableSource, node); + deleteStatement.setTableSource(from2); + node.setStatement(deleteStatement.toString()); + } + } + if(statement instanceof SQLUpdateStatement) { + SQLUpdateStatement updateStatement = (SQLUpdateStatement) statement; + tableSource = updateStatement.getTableSource(); + for (RouteResultsetNode node : rrs.getNodes()) { + SQLExprTableSource from2 = getDisTable(tableSource, node); + updateStatement.setTableSource(from2); + node.setStatement(updateStatement.toString()); + } + } + return rrs; + } - + private SQLExprTableSource getDisTable(SQLTableSource tableSource,RouteResultsetNode node) throws SQLSyntaxErrorException{ if(node.getSubTableName()==null){ String msg = " sub table not exists for " + node.getName() + " on " + tableSource; diff --git a/src/main/java/io/mycat/route/parser/druid/impl/DruidInsertParser.java b/src/main/java/io/mycat/route/parser/druid/impl/DruidInsertParser.java index 2179a2490..913691a84 100644 --- a/src/main/java/io/mycat/route/parser/druid/impl/DruidInsertParser.java +++ b/src/main/java/io/mycat/route/parser/druid/impl/DruidInsertParser.java @@ -189,6 +189,19 @@ private void parserSingleInsert(SchemaConfig schema, RouteResultset rrs, String LOGGER.warn(msg); throw new SQLNonTransientException(msg); } + if(schema.isMutilRoute()){ + for (TableConfig tableConfig:schema.getTables().values()){ + String subPartitionColumn = tableConfig.getSubPartitionColumn(); + for(int i = 0; i < insertStmt.getColumns().size(); i++) { + if(subPartitionColumn.equalsIgnoreCase(StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString()))) {//找到分片字段 + String column = StringUtil.removeBackquote(insertStmt.getColumns().get(i).toString()); + String value = StringUtil.removeBackquote(insertStmt.getValues().getValues().get(i).toString()); + ctx.getRouteCalculateUnits().get(0).addShardingExpr(tableName, column, value); + break; + } + } + } + } // insert into .... on duplicateKey //such as :INSERT INTO TABLEName (a,b,c) VALUES (1,2,3) ON DUPLICATE KEY UPDATE b=VALUES(b); //INSERT INTO TABLEName (a,b,c) VALUES (1,2,3) ON DUPLICATE KEY UPDATE c=c+1; @@ -296,6 +309,18 @@ private void parserBatchInsert(SchemaConfig schema, RouteResultset rrs, String p } else { nodes[count] = new RouteResultsetNode(tableConfig.getDataNodes().get(nodeIndex), rrs.getSqlType(),insertStmt.toString()); + if(tableConfig.isMutilRoute()){//多级路由修改表名 + String subTableName = tableConfig.getDistTables().get(nodeIndex); + nodes[count].setSubTableName(subTableName); + SQLExprTableSource tableSource = ((SQLInsertStatement) insertStmt).getTableSource(); + //getDisTable 修改表名称 + SQLIdentifierExpr sqlIdentifierExpr = new SQLIdentifierExpr(); + sqlIdentifierExpr.setParent(tableSource.getParent()); + sqlIdentifierExpr.setName(subTableName); + SQLExprTableSource from2 = new SQLExprTableSource(sqlIdentifierExpr); + ((SQLInsertStatement) insertStmt).setTableSource(from2); + nodes[count].setStatement(((SQLInsertStatement) insertStmt).toString()); + } } if(algorithm instanceof SlotFunction) { diff --git a/src/main/java/io/mycat/route/util/RouterUtil.java b/src/main/java/io/mycat/route/util/RouterUtil.java index 7afce6b4d..057b2ca73 100644 --- a/src/main/java/io/mycat/route/util/RouterUtil.java +++ b/src/main/java/io/mycat/route/util/RouterUtil.java @@ -1097,9 +1097,9 @@ public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardin routeToDistTableNode(tableName,schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect); return rrs; } - if(tableConfig.isMutilRoute()){ - routeToMutilDistTableNode(tableName,schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect); - } +// if(tableConfig.isMutilRoute()){ +// routeToMutilDistTableNode(tableName,schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect); +// } if(retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) { // mulit routes ,not cache route result if (isSelect) { @@ -1300,13 +1300,42 @@ private static RouteResultset routeToMutilDistTableNode(String tableName, Schema LOGGER.warn(msg); throw new SQLNonTransientException(msg); } - String partionCol = tableConfig.getSubPartitionColumn(); -// String primaryKey = tableConfig.getPrimaryKey(); - boolean isLoadData=false; + + + String partitionColum = tableConfig.getPartitionColumn(); + List dataNodes = new ArrayList<>(); + //计算dn节点 + for(Map.Entry>> entry : tablesAndConditions.entrySet()) { + boolean isFoundPartitionValue = partitionColum != null && entry.getValue().get(partitionColum) != null; + if(isFoundPartitionValue){ + Map> columnsMap = entry.getValue(); + Set partitionValue = columnsMap.get(partitionColum); + + for(ColumnRoutePair pair : partitionValue) { + AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm(); + if(pair.colValue != null) { + Integer tableIndex = algorithm.calculate(pair.colValue); + if(tableIndex == null) { + String msg = "can't find any valid datanode :" + tableConfig.getName() + + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue; + LOGGER.warn(msg); + throw new SQLNonTransientException(msg); + } + String dn = tableConfig.getDataNodes().get(tableIndex); + dataNodes.add(dn); + } + } + } else { + dataNodes.addAll(tableConfig.getDataNodes()); + } + } + + String subPartionCol = tableConfig.getSubPartitionColumn(); + Set tablesRouteSet = new HashSet(); - List dataNodes = tableConfig.getDataNodes(); + //所有节点的分表逻辑是一样的,所以就取一个就可以了 String dataNode = dataNodes.get(0); @@ -1317,10 +1346,10 @@ private static RouteResultset routeToMutilDistTableNode(String tableName, Schema } for(Map.Entry>> entry : tablesAndConditions.entrySet()) { - boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null; + Map> columnsMap = entry.getValue(); - Set partitionValue = columnsMap.get(partionCol); + Set partitionValue = columnsMap.get(subPartionCol); if(partitionValue == null || partitionValue.size() == 0) { tablesRouteSet.addAll(tableConfig.getDistTables()); } else { @@ -1363,12 +1392,12 @@ private static RouteResultset routeToMutilDistTableNode(String tableName, Schema //所有的dn都改写sql RouteResultsetNode[] nodes = new RouteResultsetNode[subTables.length * dataNodes.size()]; Map dataNodeSlotMap= rrs.getDataNodeSlotMap(); + int nodeIndex = 0; for(int i=0;i nodeSet = new TreeSet(); @@ -815,7 +823,17 @@ private String getPartitionColumn() { } return pColumn; } - + private String getSubPartitionColumn(){ + String pColumn; + if (tableConfig.isSecondLevel() + && tableConfig.getParentTC().getSubPartitionColumn() + .equals(tableConfig.getParentKey())) { + pColumn = tableConfig.getJoinKey(); + }else { + pColumn = tableConfig.getSubPartitionColumn(); + } + return pColumn; + } /** * 删除目录及其所有子目录和文件 * diff --git a/src/test/java/io/mycat/route/TestDisTableRuleRoute.java b/src/test/java/io/mycat/route/TestDisTableRuleRoute.java new file mode 100644 index 000000000..b99c0fb3b --- /dev/null +++ b/src/test/java/io/mycat/route/TestDisTableRuleRoute.java @@ -0,0 +1,46 @@ +package io.mycat.route; + +import io.mycat.MycatServer; +import io.mycat.SimpleCachePool; +import io.mycat.cache.LayerCachePool; +import io.mycat.config.loader.SchemaLoader; +import io.mycat.config.loader.xml.XMLSchemaLoader; +import io.mycat.config.model.SchemaConfig; +import io.mycat.config.model.SystemConfig; +import io.mycat.route.factory.RouteStrategyFactory; +import java.sql.SQLNonTransientException; +import java.util.Map; +import org.junit.Test; + +/** + * @author liunan by 2018/12/2 + */ +public class TestDisTableRuleRoute { + + + + protected Map schemaMap; + protected LayerCachePool cachePool = new SimpleCachePool(); + + public TestDisTableRuleRoute() { + String schemaFile = "/route/disRoute/schema.xml"; + String ruleFile = "/route/disRoute/rule.xml"; + SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); + schemaMap = schemaLoader.getSchemas(); + MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); + RouteStrategyFactory.init(); + } + + @Test + public void testDisTableInsert() throws SQLNonTransientException { + String sql = "insert into sqtestmonth (id,name,create_time) values(1,'sq1', '2017-5-12')"; +// String sql = "insert into sqtestmonth (name,create_time) Select name , create_time from sqtestmonth"; + SchemaConfig schema = schemaMap.get("cndb"); + RouteResultset rrs = RouteStrategyFactory + .getRouteStrategy() + .route(new SystemConfig(),schema, -1, sql, null, + null, cachePool); + System.out.println(rrs); + } + +} diff --git a/src/test/java/io/mycat/route/TestSubTableRuleRoute.java b/src/test/java/io/mycat/route/TestSubTableRuleRoute.java index a034a8d17..6461b0fd8 100644 --- a/src/test/java/io/mycat/route/TestSubTableRuleRoute.java +++ b/src/test/java/io/mycat/route/TestSubTableRuleRoute.java @@ -4,7 +4,6 @@ import io.mycat.SimpleCachePool; import io.mycat.cache.LayerCachePool; import io.mycat.config.loader.SchemaLoader; -import io.mycat.config.loader.xml.XMLRuleLoader; import io.mycat.config.loader.xml.XMLSchemaLoader; import io.mycat.config.model.SchemaConfig; import io.mycat.config.model.SystemConfig; @@ -23,8 +22,8 @@ public class TestSubTableRuleRoute { protected LayerCachePool cachePool = new SimpleCachePool(); public TestSubTableRuleRoute() { - String schemaFile = "/route/sub_tables/schema.xml"; - String ruleFile = "/route/sub_tables/rule.xml"; + String schemaFile = "/route/mulitRoute/schema.xml"; + String ruleFile = "/route/mulitRoute/rule.xml"; SchemaLoader schemaLoader = new XMLSchemaLoader(schemaFile, ruleFile); schemaMap = schemaLoader.getSchemas(); MycatServer.getInstance().getConfig().getSchemas().putAll(schemaMap); @@ -41,6 +40,19 @@ public void testSelect() throws SQLNonTransientException { Assert.assertEquals(5, rrs.getNodes().length); } + @Test + public void testUpdate() throws SQLNonTransientException { + String sql = "update sqtestmonth set name = 1 where id =1 and create_time = '2017-5-12'"; + SchemaConfig schema = schemaMap.get("cndb"); + RouteResultset rrs = RouteStrategyFactory + .getRouteStrategy() + .route(new SystemConfig(),schema, -1, sql, null, + null, cachePool); + System.out.println(rrs.getNodes()[0]); + Assert.assertTrue(rrs.getNodes()[0].getStatement().contains("sqtestmonth20175")); + Assert.assertEquals(1, rrs.getNodes().length); + } + @Test public void testInsert() throws SQLNonTransientException { String sql = "insert into sqtestmonth (id,name,create_time) values(1,'sq1', '2017-5-12')"; @@ -51,7 +63,7 @@ public void testInsert() throws SQLNonTransientException { null, cachePool); System.out.println(rrs.getNodes()[0]); Assert.assertTrue(rrs.getNodes()[0].getStatement().contains("sqtestmonth20175")); - Assert.assertEquals(2, rrs.getNodes().length); + Assert.assertEquals(1, rrs.getNodes().length); } @Test @@ -64,6 +76,6 @@ public void testSelect2() throws SQLNonTransientException { null, cachePool); System.out.println(rrs.getNodes()[0]); Assert.assertTrue(rrs.getNodes()[0].getStatement().contains("sqtestmonth20175")); - Assert.assertEquals(2, rrs.getNodes().length); + Assert.assertEquals(1, rrs.getNodes().length); } } diff --git a/src/test/resources/route/disRoute/rule.xml b/src/test/resources/route/disRoute/rule.xml new file mode 100644 index 000000000..864e4dece --- /dev/null +++ b/src/test/resources/route/disRoute/rule.xml @@ -0,0 +1,29 @@ + + + + + + + + + ID + mod-long + + + + yyyy-MM-dd + 2017-01-01 + + + + 2 + + diff --git a/src/test/resources/route/disRoute/schema.xml b/src/test/resources/route/disRoute/schema.xml new file mode 100644 index 000000000..de602b3ba --- /dev/null +++ b/src/test/resources/route/disRoute/schema.xml @@ -0,0 +1,26 @@ + + + + + + +
+ + + + + + + select user() + + + + diff --git a/src/test/resources/route/sub_tables/rule.xml b/src/test/resources/route/mulitRoute/rule.xml similarity index 100% rename from src/test/resources/route/sub_tables/rule.xml rename to src/test/resources/route/mulitRoute/rule.xml diff --git a/src/test/resources/route/sub_tables/schema.xml b/src/test/resources/route/mulitRoute/schema.xml similarity index 100% rename from src/test/resources/route/sub_tables/schema.xml rename to src/test/resources/route/mulitRoute/schema.xml