diff --git a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/ArexMocker.java b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/ArexMocker.java index ff72edd69..da1abb07a 100644 --- a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/ArexMocker.java +++ b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/ArexMocker.java @@ -16,18 +16,18 @@ public class ArexMocker implements Mocker { private long creationTime; private Mocker.Target targetRequest; private Mocker.Target targetResponse; - private transient boolean needMerge; private String operationName; - private Map tags; + private transient boolean needMerge; private final transient AtomicBoolean matched = new AtomicBoolean(false); - /** - * replay match need - */ private transient int fuzzyMatchKey; - /** - * replay match need - */ private transient int accurateMatchKey; + private transient Map eigenMap; + private Map tags; + + // original compressed text for request + private transient String request; + // original compressed text for response + private transient String response; /** * The default constructor is for deserialization @@ -103,6 +103,7 @@ public String getOperationName() { return this.operationName; } + public void setId(String id) { this.id = id; } @@ -178,4 +179,30 @@ public int getFuzzyMatchKey() { public void setFuzzyMatchKey(int fuzzyMatchKey) { this.fuzzyMatchKey = fuzzyMatchKey; } + + @Override + public Map getEigenMap() { + return eigenMap; + } + + @Override + public void setEigenMap(Map eigenMap) { + this.eigenMap = eigenMap; + } + + public String getRequest() { + return request; + } + + public void setRequest(String request) { + this.request = request; + } + + public String getResponse() { + return response; + } + + public void setResponse(String response) { + this.response = response; + } } diff --git a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/MockCategoryType.java b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/MockCategoryType.java index 2e14963f4..fd79767df 100644 --- a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/MockCategoryType.java +++ b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/MockCategoryType.java @@ -15,7 +15,7 @@ public class MockCategoryType implements Serializable { public static final MockCategoryType DYNAMIC_CLASS = createSkipComparison("DynamicClass"); public static final MockCategoryType REDIS = createSkipComparison("Redis"); public static final MockCategoryType MESSAGE_PRODUCER = createDependency("QMessageProducer"); - public static final MockCategoryType MESSAGE_CONSUMER = createEntryPoint("QMessageConsumer"); + public static final MockCategoryType MESSAGE_CONSUMER = create("QMessageConsumer",true,true); public static final MockCategoryType DUBBO_CONSUMER = createDependency("DubboConsumer"); public static final MockCategoryType DUBBO_PROVIDER = createEntryPoint("DubboProvider"); public static final MockCategoryType DUBBO_STREAM_PROVIDER = createDependency("DubboStreamProvider"); diff --git a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/Mocker.java b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/Mocker.java index 2c22d0836..3c543d161 100644 --- a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/Mocker.java +++ b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/model/Mocker.java @@ -7,6 +7,8 @@ public interface Mocker extends Serializable { + void setAppId(String appId); + String getAppId(); String getReplayId(); @@ -33,10 +35,16 @@ public interface Mocker extends Serializable { String getOperationName(); + void setOperationName(String operationName); + Target getTargetRequest(); Target getTargetResponse(); + void setTargetRequest(Target targetRequest); + + void setTargetResponse(Target targetResponse); + public static class Target implements Serializable { private String body; @@ -132,4 +140,16 @@ default String replayLogTitle() { int getFuzzyMatchKey(); void setFuzzyMatchKey(int fuzzyMatchKey); + + Map getEigenMap(); + + void setEigenMap(Map eigenMap); + + String getRequest(); + + void setRequest(String request); + + String getResponse(); + + void setResponse(String response); } diff --git a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/util/StringUtil.java b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/util/StringUtil.java index f7f8b4d66..2cd4e64da 100644 --- a/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/util/StringUtil.java +++ b/arex-agent-bootstrap/src/main/java/io/arex/agent/bootstrap/util/StringUtil.java @@ -283,11 +283,11 @@ public static String[] splitByLastSeparator(String str, char separator) { return new String[]{str.substring(0, index), str.substring(index + 1)}; } - public static int encodeAndHash(String str){ - if (isBlank(str)) { + public static int encodeAndHash(String... str){ + if (ArrayUtils.isEmpty(str)) { return 0; } - return Base64.getEncoder().encodeToString(str.getBytes(StandardCharsets.UTF_8)).hashCode(); + return Base64.getEncoder().encodeToString(String.join("_", str).getBytes(StandardCharsets.UTF_8)).hashCode(); } public static String replace(final String text, final String searchString, final String replacement) { diff --git a/arex-agent-core/src/main/java/io/arex/agent/instrumentation/BaseAgentInstaller.java b/arex-agent-core/src/main/java/io/arex/agent/instrumentation/BaseAgentInstaller.java index 01691a5dc..52d8c5291 100644 --- a/arex-agent-core/src/main/java/io/arex/agent/instrumentation/BaseAgentInstaller.java +++ b/arex-agent-core/src/main/java/io/arex/agent/instrumentation/BaseAgentInstaller.java @@ -112,7 +112,7 @@ private void initDependentComponents() { initDataCollector(); } private void initDataCollector() { - List collectorList = ServiceLoader.load(DataCollector.class, getClassLoader()); + List collectorList = ServiceLoader.load(DataCollector.class, Thread.currentThread().getContextClassLoader()); DataService.setDataCollector(collectorList); } diff --git a/arex-agent/pom.xml b/arex-agent/pom.xml index 29ee44cb0..3ec71c740 100644 --- a/arex-agent/pom.xml +++ b/arex-agent/pom.xml @@ -33,6 +33,10 @@ ${project.groupId} arex-third-party + + ${project.groupId} + arex-compare + diff --git a/arex-compare/pom.xml b/arex-compare/pom.xml new file mode 100644 index 000000000..95938a335 --- /dev/null +++ b/arex-compare/pom.xml @@ -0,0 +1,63 @@ + + + + arex-agent-parent + io.arex + ${revision} + + 4.0.0 + + arex-compare + + + + ${project.groupId} + arex-agent-bootstrap + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.slf4j + slf4j-api + + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + + + com.fasterxml.jackson + io.arex.shaded.com.fasterxml.jackson + + + + + + + + + com.fasterxml.jackson.core:** + io.arex:arex-compare + + + + + + + \ No newline at end of file diff --git a/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenCalculateHandler.java b/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenCalculateHandler.java new file mode 100644 index 000000000..93476b4f8 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenCalculateHandler.java @@ -0,0 +1,41 @@ +package io.arex.agent.compare.eigen; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.compare.handler.parse.JSONParse; +import io.arex.agent.compare.handler.parse.ObjectParse; +import io.arex.agent.compare.handler.parse.sqlparse.SqlParse; +import io.arex.agent.compare.model.RulesConfig; +import io.arex.agent.compare.model.eigen.EigenResult; + +import java.util.Objects; + +public class EigenCalculateHandler { + private static ObjectParse objectParse = new ObjectParse(); + private static JSONParse jsonParse = new JSONParse(); + private static SqlParse sqlParse = new SqlParse(); + + private static EigenMapCalculate eigenMapCalculate = new EigenMapCalculate(); + + public EigenResult doHandler(RulesConfig rulesConfig) { + Object obj; + try { + // if it is not json, it will return null + obj = objectParse.msgToObj(rulesConfig.getBaseMsg()); + + if (obj instanceof JsonNode) { + jsonParse.getJSONParseResult(obj, rulesConfig); + } + + if (Objects.equals(rulesConfig.getCategoryType(), MockCategoryType.DATABASE.getName()) + && obj instanceof ObjectNode) { + sqlParse.sqlParse((ObjectNode) obj, rulesConfig.isNameToLower()); + } + } catch (Throwable e) { + obj = null; + } + + return eigenMapCalculate.doCalculate(obj, rulesConfig); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenMapCalculate.java b/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenMapCalculate.java new file mode 100644 index 000000000..d335f7f65 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/eigen/EigenMapCalculate.java @@ -0,0 +1,155 @@ +package io.arex.agent.compare.eigen; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.arex.agent.compare.handler.log.filterrules.TimePrecisionFilter; +import io.arex.agent.compare.model.RulesConfig; +import io.arex.agent.compare.utils.IgnoreUtil; +import io.arex.agent.compare.model.eigen.EigenResult; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class EigenMapCalculate { + + private static final int OFFSET_BASIS = 0x811C9DC5; // 2166136261 + private static final int FNV_PRIME = 16777619; + + private static TimePrecisionFilter timePrecisionFilter = new TimePrecisionFilter(); + + public EigenResult doCalculate(Object obj, RulesConfig rulesConfig) { + EigenResult eigenResult = new EigenResult(); + Map eigenMap = new HashMap<>(); + if (obj == null || obj instanceof String) { + eigenMap.put(0, this.valueHash(rulesConfig.getBaseMsg())); + eigenResult.setEigenMap(eigenMap); + return eigenResult; + } + + CalculateContext calculateContext = new CalculateContext(); + calculateContext.ignoreNodeSet = rulesConfig.getIgnoreNodeSet(); + calculateContext.exclusions = rulesConfig.getExclusions(); + calculateContext.nodePath = new LinkedList<>(); + doCalculateJsonNode(obj, calculateContext, eigenMap); + eigenResult.setEigenMap(eigenMap); + return eigenResult; + } + + + private void doCalculateJsonNode(Object obj, CalculateContext calculateContext, + Map eigenMap) { + + // ignore by node name and node path + if (IgnoreUtil.ignoreProcessor(calculateContext.nodePath, calculateContext.exclusions, + calculateContext.ignoreNodeSet)) { + return; + } + + if (obj instanceof ObjectNode) { + ObjectNode objectNode = (ObjectNode) obj; + Iterator stringIterator = objectNode.fieldNames(); + while (stringIterator.hasNext()) { + String fieldName = stringIterator.next(); + JsonNode jsonNode = objectNode.get(fieldName); + + int lastHash = calculateContext.lastHash; + calculateContext.lastHash = this.pathHashWithLastHash(fieldName, lastHash); + calculateContext.nodePath.addLast(fieldName); + this.doCalculateJsonNode(jsonNode, calculateContext, eigenMap); + calculateContext.nodePath.removeLast(); + calculateContext.lastHash = lastHash; + } + + + } else if (obj instanceof ArrayNode) { + ArrayNode arrayNode = (ArrayNode) obj; + for (int i = 0; i < arrayNode.size(); i++) { + JsonNode jsonNode = arrayNode.get(i); + int lastHash = calculateContext.lastHash; + this.doCalculateJsonNode(jsonNode, calculateContext, eigenMap); + calculateContext.lastHash = lastHash; + } + } else { + // calculate eigen value + String value = obj == null ? null : obj.toString(); + + // process time + Instant instant = timePrecisionFilter.identifyTime(value); + if (instant != null) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm") + .withZone(ZoneId.systemDefault()); + value = formatter.format(instant); + } + int pathHash = calculateContext.lastHash; + long valueHash = this.valueHash(value); + eigenMap.put(pathHash, eigenMap.getOrDefault(pathHash, 0L) + valueHash); + } + } + + private int pathHashWithLastHash(String nodeName, int lastHash) { + int key = lastHash; + for (byte c : nodeName.getBytes()) { + key = (key ^ c) * FNV_PRIME; + } + return Math.abs(key); + } + + // FNV-1a hash function, think about null and empty string + private long valueHash(String value) { + if (value == null) { + return 1; + } + if (value.isEmpty()) { + return 2; + } + + int key = OFFSET_BASIS; + for (byte c : value.getBytes()) { + key = (key ^ c) * FNV_PRIME; + } + key += key << 13; + key ^= key >> 7; + key += key << 3; + key ^= key >> 17; + key += key << 5; + return Math.abs(key); + } + + + private static class CalculateContext { + + private LinkedList nodePath; + + private int lastHash = OFFSET_BASIS; + + private Set ignoreNodeSet; + private List> exclusions; + + public CalculateContext() { + } + + public LinkedList getNodePath() { + return nodePath; + } + + public void setNodePath(LinkedList nodePath) { + this.nodePath = nodePath; + } + + public int getLastHash() { + return lastHash; + } + + public void setLastHash(int lastHash) { + this.lastHash = lastHash; + } + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/log/filterrules/TimePrecisionFilter.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/log/filterrules/TimePrecisionFilter.java new file mode 100644 index 000000000..a17f67501 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/log/filterrules/TimePrecisionFilter.java @@ -0,0 +1,139 @@ +package io.arex.agent.compare.handler.log.filterrules; + +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; + +public class TimePrecisionFilter { + private static AbstractDataProcessor dataProcessor; + + private static DateTimeFormatter parseFormat1 = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd") + .optionalStart().appendLiteral(' ').optionalEnd() + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS")) + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSS")) + .toFormatter(); + private static DateTimeFormatter parseFormat2 = new DateTimeFormatterBuilder() + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSSSSS")) + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSS")) + .toFormatter(); + private static DateTimeFormatter parseFormat3 = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd") + .optionalStart().appendLiteral('T').optionalEnd() + .optionalStart().appendLiteral(' ').optionalEnd() + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSSXXX")) + .appendOptional(DateTimeFormatter.ofPattern("HH:mm:ss.SSSZ")) + .toFormatter(); + + static { + parseFormat1 = parseFormat1.withZone(ZoneId.of("UTC")); + dataProcessor = new ProcessorChainBuilder() + .addProcessor(new FirstDataProcessor()) + .addProcessor(new SecondDataProcessor()) + .addProcessor(new ThirdDataProcessor()) + .build(); + } + + public Instant identifyTime(String data) { + if (data == null || data.length() < 12 || data.length() > 29) { + return null; + } + + if ((data.startsWith("0") || data.startsWith("1") || data.startsWith("2"))) { + Instant baseTime = dataProcessor.process(data); + if (baseTime == null) { + return null; + } + return baseTime; + } + return null; + } + + private static abstract class AbstractDataProcessor { + + private AbstractDataProcessor nextProcessor; + + public void setNextProcessor(AbstractDataProcessor nextProcessor) { + this.nextProcessor = nextProcessor; + } + + public Instant process(String data) { + Instant date = processData(data); + if (date != null) { + return date; + } + if (this.nextProcessor == null) { + return null; + } + return this.nextProcessor.process(data); + } + + protected abstract Instant processData(String data); + } + + private static class FirstDataProcessor extends AbstractDataProcessor { + + @Override + protected Instant processData(String data) { + + Instant instant = null; + try { + ZonedDateTime zdt = ZonedDateTime.parse(data, parseFormat1); + instant = zdt.toInstant(); + } catch (Exception e) { + } + return instant; + } + } + + private static class SecondDataProcessor extends AbstractDataProcessor { + + @Override + protected Instant processData(String data) { + Instant instant = null; + try { + LocalTime time = LocalTime.parse(data, parseFormat2); + LocalDate date = LocalDate.ofEpochDay(0); + LocalDateTime dateTime = LocalDateTime.of(date, time); + instant = dateTime.toInstant(ZoneOffset.UTC); + } catch (Exception e) { + } + return instant; + + } + } + + private static class ThirdDataProcessor extends AbstractDataProcessor { + + @Override + protected Instant processData(String data) { + Instant instant = null; + try { + ZonedDateTime zdt = ZonedDateTime.parse(data, parseFormat3); + instant = zdt.toInstant(); + } catch (Exception e) { + } + return instant; + } + } + + private static class ProcessorChainBuilder { + + private AbstractDataProcessor firstProcessor; + private AbstractDataProcessor lastProcessor; + + public ProcessorChainBuilder addProcessor(AbstractDataProcessor processor) { + if (firstProcessor == null) { + firstProcessor = processor; + } else { + lastProcessor.setNextProcessor(processor); + } + lastProcessor = processor; + return this; + } + + public AbstractDataProcessor build() { + return firstProcessor; + } + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/JSONParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/JSONParse.java new file mode 100644 index 000000000..a507ab132 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/JSONParse.java @@ -0,0 +1,22 @@ +package io.arex.agent.compare.handler.parse; + +import io.arex.agent.compare.model.RulesConfig; +import io.arex.agent.compare.model.log.NodeEntity; +import io.arex.agent.compare.utils.NameConvertUtil; + +import java.util.List; +import java.util.Map; + +public class JSONParse { + public Map, String> getJSONParseResult(Object obj, RulesConfig rulesConfig) { + + StringAndCompressParse stringAndCompressParse = new StringAndCompressParse(); + stringAndCompressParse.setNameToLower(rulesConfig.isNameToLower()); + stringAndCompressParse.getJSONParse(obj, obj); + // Convert field names in JSONObject to lowercase + if (rulesConfig.isNameToLower()) { + NameConvertUtil.nameConvert(obj); + } + return stringAndCompressParse.getOriginal(); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/ObjectParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/ObjectParse.java new file mode 100644 index 000000000..db5dc9231 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/ObjectParse.java @@ -0,0 +1,31 @@ +package io.arex.agent.compare.handler.parse; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.agent.compare.utils.JacksonHelperUtil; + + +public class ObjectParse { + + public Object msgToObj(String msg) { + if (StringUtil.isEmpty(msg)) { + return msg; + } + + Object obj; + try { + if (msg.startsWith("[")) { + obj = JacksonHelperUtil.objectMapper.readValue(msg, ArrayNode.class); + } else if (msg.startsWith("{")) { + obj = JacksonHelperUtil.objectMapper.readValue(msg, ObjectNode.class); + } else { + obj = msg; + } + } catch (RuntimeException | JsonProcessingException e) { + obj = msg; + } + return obj; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/StringAndCompressParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/StringAndCompressParse.java new file mode 100644 index 000000000..fabd284c0 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/StringAndCompressParse.java @@ -0,0 +1,106 @@ +package io.arex.agent.compare.handler.parse; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.arex.agent.bootstrap.internal.Pair; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.agent.compare.model.log.NodeEntity; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import io.arex.agent.compare.utils.ListUtil; + +import java.util.*; + +public class StringAndCompressParse { + private List currentNode = new ArrayList<>(); + + private Map, String> original = new HashMap<>(); + + private boolean nameToLower; + + public Map, String> getOriginal() { + return original; + } + + public void setNameToLower(boolean nameToLower) { + this.nameToLower = nameToLower; + } + + public void getJSONParse(Object obj, Object preObj) { + if (obj == null || obj instanceof NullNode) { + return; + } + + if (obj instanceof ObjectNode) { + ObjectNode jsonObject = (ObjectNode) obj; + List names = JacksonHelperUtil.getNames(jsonObject); + for (String fieldName : names) { + currentNode.add(new NodeEntity(fieldName, 0)); + Object objFieldValue = jsonObject.get(fieldName); + getJSONParse(objFieldValue, obj); + ListUtil.removeLast(currentNode); + } + } else if (obj instanceof ArrayNode) { + ArrayNode objArray = (ArrayNode) obj; + for (int i = 0; i < objArray.size(); i++) { + currentNode.add(new NodeEntity(null, i)); + Object element = objArray.get(i); + getJSONParse(element, obj); + ListUtil.removeLast(currentNode); + } + + } else { + String value = ((JsonNode) obj).asText(); + Pair objectBooleanPair = processStringParse(value, preObj); + if (objectBooleanPair.getFirst() == null) { + return; + } + if (Objects.equals(objectBooleanPair.getSecond(), Boolean.TRUE)) { + getJSONParse(objectBooleanPair.getFirst(), preObj); + } + + String currentName = getCurrentName(currentNode); + if (preObj instanceof ObjectNode) { + ((ObjectNode) preObj).set(currentName, objectBooleanPair.getFirst()); + original.put(new ArrayList<>(currentNode), value); + } else if (preObj instanceof ArrayNode) { + ((ArrayNode) preObj).set(Integer.parseInt(currentName), objectBooleanPair.getFirst()); + original.put(new ArrayList<>(currentNode), value); + } + } + + } + + private String getCurrentName(List currentNode) { + NodeEntity nodeEntity = currentNode.get(currentNode.size() - 1); + if (nodeEntity.getNodeName() != null) { + return nodeEntity.getNodeName(); + } else { + return String.valueOf(nodeEntity.getIndex()); + } + } + + private Pair processStringParse(String value, Object preObj) { + + JsonNode objTemp = null; + + if (StringUtil.isEmpty(value)) { + return Pair.of(null, Boolean.FALSE); + } + + if (value.startsWith("{") && value.endsWith("}")) { + try { + objTemp = JacksonHelperUtil.objectMapper.readValue(value, ObjectNode.class); + } catch (JsonProcessingException e) { + } + } else if (value.startsWith("[") && value.endsWith("]")) { + try { + objTemp = JacksonHelperUtil.objectMapper.readValue(value, ArrayNode.class); + } catch (JsonProcessingException e) { + } + } + return objTemp == null ? Pair.of(null, Boolean.FALSE) : Pair.of(objTemp, Boolean.TRUE); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/Parse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/Parse.java new file mode 100644 index 000000000..5c79cb652 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/Parse.java @@ -0,0 +1,8 @@ +package io.arex.agent.compare.handler.parse.sqlparse; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public interface Parse { + + ObjectNode parse(T parseObj); +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/SqlParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/SqlParse.java new file mode 100644 index 000000000..229e17d81 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/SqlParse.java @@ -0,0 +1,123 @@ +package io.arex.agent.compare.handler.parse.sqlparse; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.arex.agent.bootstrap.internal.Pair; +import io.arex.agent.compare.handler.parse.sqlparse.action.ActionFactory; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import io.arex.agent.compare.utils.LogUtil; +import io.arex.agent.compare.utils.NameConvertUtil; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.select.Select; + +import java.util.ArrayList; +import java.util.List; + +public class SqlParse { + // if return null, indicate the sql parsed fail. + public ParsedResult sqlParse(ObjectNode jsonObj, boolean nameToLower) { + JsonNode databaseBody = jsonObj.get(DbParseConstants.BODY); + if (databaseBody == null) { + return new ParsedResult(null, false); + } + + boolean successParse = true; + ArrayNode parsedSql = JacksonHelperUtil.getArrayNode(); + List isSelect = new ArrayList<>(); + try { + if (databaseBody instanceof TextNode) { + Pair tempMutablePair = sqlParse(databaseBody.asText()); + parsedSql.add(tempMutablePair.getFirst()); + isSelect.add(tempMutablePair.getSecond()); + } else if (databaseBody instanceof ArrayNode) { + ArrayNode databaseBodyArray = (ArrayNode) databaseBody; + for (int i = 0; i < databaseBodyArray.size(); i++) { + Pair tempMutablePair = sqlParse(databaseBodyArray.get(i).asText()); + parsedSql.add(tempMutablePair.getFirst()); + isSelect.add(tempMutablePair.getSecond()); + } + } else { + successParse = false; + } + } catch (Throwable throwable) { + LogUtil.warn("arex sqlParse error", throwable); + successParse = false; + } + + ParsedResult result = new ParsedResult(); + if (!successParse) { + this.fillOriginalSql(jsonObj, databaseBody); + result.setSuccess(false); + } else { + if (nameToLower) { + NameConvertUtil.nameConvert(parsedSql); + } + jsonObj.set(DbParseConstants.PARSED_SQL, parsedSql); + result.setSuccess(true); + result.setIsSelect(isSelect); + } + return result; + } + + @SuppressWarnings("unchecked") + public Pair sqlParse(String sql) throws JSQLParserException { + if (sql == null || sql.length() > DbParseConstants.SQL_LENGTH_LIMIT) { + throw new RuntimeException("sql is null or too long"); + } + Statement statement = CCJSqlParserUtil.parse(sql); + Parse parse = ActionFactory.selectParse(statement); + return Pair.of(parse.parse(statement), statement instanceof Select); + } + + private void fillOriginalSql(ObjectNode objectNode, JsonNode databaseBody) { + ObjectNode backUpObj = JacksonHelperUtil.getObjectNode(); + backUpObj.set(DbParseConstants.ORIGINAL_SQL, databaseBody); + ArrayNode parsedSql = JacksonHelperUtil.getArrayNode(); + parsedSql.add(backUpObj); + objectNode.set(DbParseConstants.PARSED_SQL, parsedSql); + } + + private static class ParsedResult { + + private List isSelect; + private boolean success; + + public ParsedResult() { + + } + + public ParsedResult(boolean success) { + this.success = success; + } + + public ParsedResult(List isSelect) { + this.isSelect = isSelect; + } + + public ParsedResult(List isSelect, boolean success) { + this.isSelect = isSelect; + this.success = success; + } + + public boolean isSuccess() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + public List getIsSelect() { + return isSelect; + } + + public void setIsSelect(List isSelect) { + this.isSelect = isSelect; + } + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ActionFactory.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ActionFactory.java new file mode 100644 index 000000000..2d195b60f --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ActionFactory.java @@ -0,0 +1,31 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.delete.Delete; +import net.sf.jsqlparser.statement.execute.Execute; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.replace.Replace; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.update.Update; + +public class ActionFactory { + + public static Parse selectParse(Statement statement) { + if (statement instanceof Insert) { + return new InsertParse(); + } else if (statement instanceof Delete) { + return new DeleteParse(); + } else if (statement instanceof Update) { + return new UpdateParse(); + } else if (statement instanceof Select) { + return new SelectParse(); + } else if (statement instanceof Replace) { + return new ReplaceParse(); + } else if (statement instanceof Execute) { + return new ExecuteParse(); + } else { + throw new UnsupportedOperationException("not support"); + } + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/DeleteParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/DeleteParse.java new file mode 100644 index 000000000..86f0add48 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/DeleteParse.java @@ -0,0 +1,88 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; + +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexExpressionVisitorAdapter; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexOrderByVisitorAdapter; +import io.arex.agent.compare.handler.parse.sqlparse.select.utils.JoinParseUtil; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.delete.Delete; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.Limit; +import net.sf.jsqlparser.statement.select.OrderByElement; + +/** + * the example of parsed delete sql: { "action": "DELETE", "table": + * "Exam", "where": { "andor": [ "and", "and" ], "columns": { "a.rnk <= 3": "", "a.per_id in (select + * per_id from colle_subject)": "" } } } + */ +public class DeleteParse implements Parse { + + @Override + public ObjectNode parse(Delete parseObj) { + ObjectNode sqlObject = JacksonHelperUtil.getObjectNode(); + sqlObject.put(DbParseConstants.ACTION, DbParseConstants.DELETE); + + // tables parse + List tables = parseObj.getTables(); + if (tables != null && !tables.isEmpty()) { + ObjectNode delTableObj = JacksonHelperUtil.getObjectNode(); + tables.forEach(item -> { + delTableObj.put(item.getFullyQualifiedName(), DbParseConstants.EMPTY); + }); + sqlObject.set(DbParseConstants.DEL_TABLES, delTableObj); + } + + // table parse + Table table = parseObj.getTable(); + if (table != null) { + sqlObject.put(DbParseConstants.TABLE, table.getFullyQualifiedName()); + } + + // join parse + List joins = parseObj.getJoins(); + if (joins != null && !joins.isEmpty()) { + ArrayNode joinArr = JacksonHelperUtil.getArrayNode(); + joins.forEach(item -> { + joinArr.add(JoinParseUtil.parse(item)); + }); + sqlObject.set(DbParseConstants.JOIN, joinArr); + } + + // where parse + Expression where = parseObj.getWhere(); + if (where != null) { + ObjectNode whereObj = JacksonHelperUtil.getObjectNode(); + whereObj.set(DbParseConstants.AND_OR, JacksonHelperUtil.getArrayNode()); + whereObj.set(DbParseConstants.COLUMNS, JacksonHelperUtil.getObjectNode()); + + where.accept(new ArexExpressionVisitorAdapter(whereObj)); + sqlObject.set(DbParseConstants.WHERE, whereObj); + } + + // orderby parse + List orderByElements = parseObj.getOrderByElements(); + if (orderByElements != null && !orderByElements.isEmpty()) { + ObjectNode orderByObj = JacksonHelperUtil.getObjectNode(); + ArexOrderByVisitorAdapter arexOrderByVisitorAdapter = new ArexOrderByVisitorAdapter( + orderByObj); + orderByElements.forEach(item -> { + item.accept(arexOrderByVisitorAdapter); + }); + sqlObject.set(DbParseConstants.ORDER_BY, orderByObj); + } + + // limit parse + Limit limit = parseObj.getLimit(); + if (limit != null) { + sqlObject.put(DbParseConstants.LIMIT, limit.toString()); + } + return sqlObject; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ExecuteParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ExecuteParse.java new file mode 100644 index 000000000..4d00671d0 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ExecuteParse.java @@ -0,0 +1,50 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; + +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexExpressionVisitorAdapter; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.execute.Execute; + +public class ExecuteParse implements Parse { + + @Override + public ObjectNode parse(Execute parseObj) { + ObjectNode sqlObject = JacksonHelperUtil.getObjectNode(); + sqlObject.put(DbParseConstants.ACTION, DbParseConstants.EXECUTE); + + // execute name parse + String executeName = parseObj.getName(); + if (executeName != null) { + sqlObject.put(DbParseConstants.EXECUTE_NAME, executeName); + } + + // expressions parse + ExpressionList exprList = parseObj.getExprList(); + if (exprList != null) { + + List expressions = exprList.getExpressions(); + + if (expressions != null && !expressions.isEmpty()) { + ArrayNode sqlColumnArr = JacksonHelperUtil.getArrayNode(); + + ObjectNode setColumnObj = JacksonHelperUtil.getObjectNode(); + setColumnObj.set(DbParseConstants.AND_OR, JacksonHelperUtil.getArrayNode()); + setColumnObj.set(DbParseConstants.COLUMNS, JacksonHelperUtil.getObjectNode()); + for (Expression expression : expressions) { + expression.accept(new ArexExpressionVisitorAdapter(setColumnObj)); + } + sqlColumnArr.add(setColumnObj.get(DbParseConstants.COLUMNS)); + sqlObject.set(DbParseConstants.COLUMNS, sqlColumnArr); + } + } + + return sqlObject; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/InsertParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/InsertParse.java new file mode 100644 index 000000000..7a3f259d6 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/InsertParse.java @@ -0,0 +1,78 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexItemsListVisitorAdapter; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.insert.Insert; + +import java.util.List; + +public class InsertParse implements Parse { + + @Override + public ObjectNode parse(Insert parseObj) { + ObjectNode sqlObject = JacksonHelperUtil.getObjectNode(); + sqlObject.put(DbParseConstants.ACTION, DbParseConstants.INSERT); + + // table parse + Table table = parseObj.getTable(); + if (table != null) { + sqlObject.put(DbParseConstants.TABLE, table.getFullyQualifiedName()); + } + + // columns parse + List columns = parseObj.getColumns(); + if (columns != null && !columns.isEmpty()) { + ArrayNode sqlColumnArr = JacksonHelperUtil.getArrayNode(); + ArrayNode values = JacksonHelperUtil.getArrayNode(); + ItemsList itemsList = parseObj.getItemsList(); + itemsList.accept(new ArexItemsListVisitorAdapter(values)); + for (int i = 0; i < values.size(); i++) { + ObjectNode sqlColumnItem = JacksonHelperUtil.getObjectNode(); + ArrayNode columnValueArray = (ArrayNode) values.get(i); + int columnValueSize = columnValueArray.size(); + for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { + JsonNode value = new TextNode("?"); + if (columnIndex < columnValueSize) { + value = columnValueArray.get(columnIndex); + } + sqlColumnItem.set(columns.get(columnIndex).toString(), value); + } + sqlColumnArr.add(sqlColumnItem); + } + sqlObject.set(DbParseConstants.COLUMNS, sqlColumnArr); + } + + // setColumns parse + List setColumns = parseObj.getSetColumns(); + if (setColumns != null && !setColumns.isEmpty()) { + ArrayNode sqlColumnArr = JacksonHelperUtil.getArrayNode(); + ObjectNode setColumnObj = JacksonHelperUtil.getObjectNode(); + ArrayNode values = JacksonHelperUtil.getArrayNode(); + List setExpressionList = parseObj.getSetExpressionList(); + for (Expression expression : setExpressionList) { + values.add(expression.toString()); + } + for (int i = 0; i < setColumns.size(); i++) { + Object value = "?"; + if (i < values.size()) { + value = values.get(i); + } + setColumnObj.putPOJO(setColumns.get(i).toString(), value); + } + sqlColumnArr.add(setColumnObj); + sqlObject.set(DbParseConstants.COLUMNS, sqlColumnArr); + } + + return sqlObject; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ReplaceParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ReplaceParse.java new file mode 100644 index 000000000..f435b4f0d --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/ReplaceParse.java @@ -0,0 +1,79 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.List; + +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexItemsListVisitorAdapter; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.schema.Table; +import net.sf.jsqlparser.statement.replace.Replace; + +public class ReplaceParse implements Parse { + + @Override + public ObjectNode parse(Replace parseObj) { + ObjectNode sqlObject = JacksonHelperUtil.getObjectNode(); + sqlObject.put(DbParseConstants.ACTION, DbParseConstants.REPLACE); + + // table parse + Table table = parseObj.getTable(); + if (table != null) { + sqlObject.put(DbParseConstants.TABLE, table.getFullyQualifiedName()); + } + + // columns parse + List columns = parseObj.getColumns(); + if (columns != null && !columns.isEmpty()) { + ArrayNode sqlColumnArr = JacksonHelperUtil.getArrayNode(); + ArrayNode values = JacksonHelperUtil.getArrayNode(); + ItemsList itemsList = parseObj.getItemsList(); + if (itemsList != null) { + itemsList.accept(new ArexItemsListVisitorAdapter(values)); + for (int i = 0; i < values.size(); i++) { + ObjectNode sqlColumnItem = JacksonHelperUtil.getObjectNode(); + ArrayNode columnValueArray = (ArrayNode) values.get(i); + int columnValueSize = columnValueArray.size(); + for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { + JsonNode value = new TextNode("?"); + if (columnIndex < columnValueSize) { + value = columnValueArray.get(columnIndex); + } + sqlColumnItem.set(columns.get(columnIndex).toString(), value); + } + sqlColumnArr.add(sqlColumnItem); + } + sqlObject.set(DbParseConstants.COLUMNS, sqlColumnArr); + } + } + + // expressions parse + List expressions = parseObj.getExpressions(); + if (expressions != null && !expressions.isEmpty()) { + ArrayNode sqlColumnArr = JacksonHelperUtil.getArrayNode(); + ObjectNode setColumnObj = JacksonHelperUtil.getObjectNode(); + ArrayNode values = JacksonHelperUtil.getArrayNode(); + for (Expression expression : expressions) { + values.add(expression.toString()); + } + for (int i = 0; i < columns.size(); i++) { + Object value = "?"; + if (i < values.size()) { + value = values.get(i); + } + setColumnObj.putPOJO(columns.get(i).toString(), value); + } + sqlColumnArr.add(setColumnObj); + sqlObject.set(DbParseConstants.COLUMNS, sqlColumnArr); + } + + return sqlObject; + } +} \ No newline at end of file diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/SelectParse.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/SelectParse.java new file mode 100644 index 000000000..b1092c645 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/action/SelectParse.java @@ -0,0 +1,30 @@ +package io.arex.agent.compare.handler.parse.sqlparse.action; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.arex.agent.compare.handler.parse.sqlparse.Parse; +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.handler.parse.sqlparse.select.ArexSelectVisitorAdapter; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; + +/** + * the example of parsed select sql: { "action": "SELECT", "columns": + * { "a.Salary": "" }, "from": { "table": { "action": "SELECT", "columns": { "*": "", "dense_rank() + * over(partition by departmentid order by Salary desc) as rnk": "" }, "from": { "table": [ + * "Employee" ] } }, "alias": "a" }, "join": [ { "type": "LEFT join", "table": "department b", "on": + * { "a.departmentid = b.Id AND a.aa = b.aa": "" } } ], "where": { "andor": [ "and", "and" ], + * "columns": { "a.rnk <= 3": "", "a.per_id in (select per_id from colle_subject)": "" } } } + */ +public class SelectParse implements Parse
intoTables = plainSelect.getIntoTables(); + if (intoTables != null && !intoTables.isEmpty()) { + sqlObj.put(DbParseConstants.INTO, intoTables.toString()); + } + + // fromItem parse + FromItem fromItem = plainSelect.getFromItem(); + if (fromItem != null) { + ObjectNode fromObj = JacksonHelperUtil.getObjectNode(); + ArexFromItemVisitorAdapter arexFromItemVisitorAdapter = new ArexFromItemVisitorAdapter( + fromObj); + fromItem.accept(arexFromItemVisitorAdapter); + sqlObj.set(DbParseConstants.FROM, fromObj); + } + + // jonis parse + List joins = plainSelect.getJoins(); + if (joins != null && !joins.isEmpty()) { + ArrayNode joinArr = JacksonHelperUtil.getArrayNode(); + joins.forEach(item -> { + joinArr.add(JoinParseUtil.parse(item)); + }); + sqlObj.put(DbParseConstants.JOIN, joinArr); + } + + // where parse + Expression where = plainSelect.getWhere(); + if (where != null) { + // JSONObject whereObj = new JSONObject(); + ObjectNode whereObj = JacksonHelperUtil.getObjectNode(); + whereObj.set(DbParseConstants.AND_OR, JacksonHelperUtil.getArrayNode()); + whereObj.set(DbParseConstants.COLUMNS, JacksonHelperUtil.getObjectNode()); + ArexExpressionVisitorAdapter arexExpressionVisitorAdapter = new ArexExpressionVisitorAdapter( + whereObj); + where.accept(arexExpressionVisitorAdapter); + sqlObj.set(DbParseConstants.WHERE, whereObj); + } + + // group by parse + GroupByElement groupBy = plainSelect.getGroupBy(); + if (groupBy != null) { + sqlObj.put(DbParseConstants.GROUP_BY, groupBy.toString()); + } + + // having parse + Expression having = plainSelect.getHaving(); + if (having != null) { + ObjectNode havingObj = JacksonHelperUtil.getObjectNode(); + havingObj.put(DbParseConstants.AND_OR, JacksonHelperUtil.getArrayNode()); + havingObj.put(DbParseConstants.COLUMNS, JacksonHelperUtil.getObjectNode()); + ArexExpressionVisitorAdapter arexExpressionVisitorAdapter = new ArexExpressionVisitorAdapter( + havingObj); + having.accept(arexExpressionVisitorAdapter); + sqlObj.put(DbParseConstants.HAVING, havingObj); + } + + // order by parse + List orderByElements = plainSelect.getOrderByElements(); + if (orderByElements != null && !orderByElements.isEmpty()) { + ObjectNode orderByObj = JacksonHelperUtil.getObjectNode(); + ArexOrderByVisitorAdapter arexOrderByVisitorAdapter = new ArexOrderByVisitorAdapter( + orderByObj); + orderByElements.forEach(item -> { + item.accept(arexOrderByVisitorAdapter); + }); + sqlObj.put(DbParseConstants.ORDER_BY, orderByObj); + } + + // fetch parse + Fetch fetch = plainSelect.getFetch(); + if (fetch != null) { + sqlObj.put(DbParseConstants.FETCH, fetch.toString()); + } + // optimizeFor parse + OptimizeFor optimizeFor = plainSelect.getOptimizeFor(); + if (optimizeFor != null) { + sqlObj.put(DbParseConstants.OPTIMIZE_FOR, optimizeFor.toString()); + } + + // limit parse + Limit limit = plainSelect.getLimit(); + if (limit != null) { + sqlObj.put(DbParseConstants.LIMIT, limit.toString()); + } + + // offset parse + Offset offset = plainSelect.getOffset(); + if (offset != null) { + sqlObj.put(DbParseConstants.OFFSET, offset.toString()); + } + + // forUpdate parse + boolean forUpdate = plainSelect.isForUpdate(); + if (forUpdate) { + sqlObj.put(DbParseConstants.FOR_UPDATE, true); + } + + // forUpdateTable parse + Table forUpdateTable = plainSelect.getForUpdateTable(); + if (forUpdateTable != null) { + sqlObj.put(DbParseConstants.FOR_UPDATE_TABLE, forUpdateTable.toString()); + } + + // noWait parse + boolean noWait = plainSelect.isNoWait(); + if (noWait) { + sqlObj.put(DbParseConstants.NO_WAIT, true); + } + + // wait parse + Wait wait = plainSelect.getWait(); + if (wait != null) { + sqlObj.put(DbParseConstants.WAIT, wait.toString()); + } + + } + + @Override + public void visit(SetOperationList setOperationList) { + sqlObj.put("setOperationList", setOperationList.toString()); + } + + @Override + public void visit(WithItem withItem) { + sqlObj.put("withItem", withItem.toString()); + } + + @Override + public void visit(ValuesStatement valuesStatement) { + sqlObj.put("valuesStatement", valuesStatement.toString()); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/select/utils/JoinParseUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/select/utils/JoinParseUtil.java new file mode 100644 index 000000000..9260fa8e5 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/handler/parse/sqlparse/select/utils/JoinParseUtil.java @@ -0,0 +1,90 @@ +package io.arex.agent.compare.handler.parse.sqlparse.select.utils; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Collection; +import java.util.List; + +import io.arex.agent.compare.handler.parse.sqlparse.constants.DbParseConstants; +import io.arex.agent.compare.utils.JacksonHelperUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.FromItem; +import net.sf.jsqlparser.statement.select.Join; + +public class JoinParseUtil { + + public static ObjectNode parse(Join parseObj) { + + ObjectNode res = JacksonHelperUtil.getObjectNode(); + // join type parse + res.put(DbParseConstants.TYPE, getJOINType(parseObj)); + + // rightItem parse + FromItem rightItem = parseObj.getRightItem(); + if (rightItem != null) { + res.put(DbParseConstants.TABLE, rightItem.toString()); + } + + // onExpressions parse + Collection onExpressions = parseObj.getOnExpressions(); + if (onExpressions != null && !onExpressions.isEmpty()) { + ObjectNode onObj = JacksonHelperUtil.getObjectNode(); + onExpressions.forEach(item -> { + onObj.put(item.toString(), DbParseConstants.EMPTY); + }); + res.put(DbParseConstants.ON, onObj); + } + + // usingColumns parse + List usingColumns = parseObj.getUsingColumns(); + if (usingColumns != null && !usingColumns.isEmpty()) { + ObjectNode usingObj = JacksonHelperUtil.getObjectNode(); + usingColumns.forEach(item -> { + usingObj.put(item.toString(), DbParseConstants.EMPTY); + }); + res.put(DbParseConstants.USING, usingObj); + } + + return res; + } + + private static String getJOINType(Join parseObj) { + StringBuilder builder = new StringBuilder(); + if (parseObj.isSimple() && parseObj.isOuter()) { + builder.append("OUTER JOIN"); + } else if (parseObj.isSimple()) { + builder.append(""); + } else { + if (parseObj.isNatural()) { + builder.append("NATURAL "); + } + + if (parseObj.isRight()) { + builder.append("RIGHT "); + } else if (parseObj.isFull()) { + builder.append("FULL "); + } else if (parseObj.isLeft()) { + builder.append("LEFT "); + } else if (parseObj.isCross()) { + builder.append("CROSS "); + } + + if (parseObj.isOuter()) { + builder.append("OUTER "); + } else if (parseObj.isInner()) { + builder.append("INNER "); + } else if (parseObj.isSemi()) { + builder.append("SEMI "); + } + + if (parseObj.isStraight()) { + builder.append("STRAIGHT_JOIN "); + } else if (parseObj.isApply()) { + builder.append("APPLY "); + } else { + builder.append("JOIN"); + } + } + return builder.toString(); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/model/RulesConfig.java b/arex-compare/src/main/java/io/arex/agent/compare/model/RulesConfig.java new file mode 100644 index 000000000..fdb991112 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/model/RulesConfig.java @@ -0,0 +1,59 @@ +package io.arex.agent.compare.model; + +import java.util.List; +import java.util.Set; + +public class RulesConfig { + private String categoryType; + + private String baseMsg; + + private List> exclusions; + + private Set ignoreNodeSet; + + private boolean nameToLower; + + public RulesConfig() { + } + + public String getCategoryType() { + return categoryType; + } + + public void setCategoryType(String categoryType) { + this.categoryType = categoryType; + } + + public String getBaseMsg() { + return baseMsg; + } + + public void setBaseMsg(String baseMsg) { + this.baseMsg = baseMsg; + } + + public List> getExclusions() { + return exclusions; + } + + public void setExclusions(List> exclusions) { + this.exclusions = exclusions; + } + + public Set getIgnoreNodeSet() { + return ignoreNodeSet; + } + + public void setIgnoreNodeSet(Set ignoreNodeSet) { + this.ignoreNodeSet = ignoreNodeSet; + } + + public boolean isNameToLower() { + return nameToLower; + } + + public void setNameToLower(boolean nameToLower) { + this.nameToLower = nameToLower; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenOptions.java b/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenOptions.java new file mode 100644 index 000000000..c0384b8ba --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenOptions.java @@ -0,0 +1,46 @@ +package io.arex.agent.compare.model.eigen; + +import java.util.List; +import java.util.Set; + +public class EigenOptions { + + private String categoryType; + + /** + * the collection of the node name which is ignore + */ + private Set ignoreNodes; + + /** + * the collection of the node path which is ignore + */ + private Set> exclusions; + + public EigenOptions() { + } + + public void setCategoryType(String categoryType) { + this.categoryType = categoryType; + } + + public void setIgnoreNodes(Set ignoreNodes) { + this.ignoreNodes = ignoreNodes; + } + + public void setExclusions(Set> exclusions) { + this.exclusions = exclusions; + } + + public String getCategoryType() { + return categoryType; + } + + public Set> getExclusions() { + return exclusions; + } + + public Set getIgnoreNodes() { + return ignoreNodes; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenResult.java b/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenResult.java new file mode 100644 index 000000000..4ddeb93bf --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/model/eigen/EigenResult.java @@ -0,0 +1,15 @@ +package io.arex.agent.compare.model.eigen; + +import java.util.Map; + +public class EigenResult { + private Map eigenMap; + + public Map getEigenMap() { + return eigenMap; + } + + public void setEigenMap(Map eigenMap) { + this.eigenMap = eigenMap; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/model/enumeration/Constant.java b/arex-compare/src/main/java/io/arex/agent/compare/model/enumeration/Constant.java new file mode 100644 index 000000000..f49fb8afe --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/model/enumeration/Constant.java @@ -0,0 +1,6 @@ +package io.arex.agent.compare.model.enumeration; + +public interface Constant { + + String DYNAMIC_PATH = "*"; +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/model/log/NodeEntity.java b/arex-compare/src/main/java/io/arex/agent/compare/model/log/NodeEntity.java new file mode 100644 index 000000000..e3304e196 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/model/log/NodeEntity.java @@ -0,0 +1,68 @@ +package io.arex.agent.compare.model.log; + +import java.util.Objects; + +public class NodeEntity { + + private String nodeName; + private int index; + + public NodeEntity() { + + } + + public NodeEntity(String nodeName, int index) { + this.nodeName = nodeName; + this.index = index; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + @Override + public int hashCode() { + int result = index; + result = 31 * result + (nodeName != null ? nodeName.hashCode() : 0); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof NodeEntity)) { + return false; + } + + NodeEntity that = (NodeEntity) obj; + if (Objects.equals(this.getNodeName(), that.getNodeName()) + && this.getIndex() == that.getIndex()) { + return true; + } + return false; + } + + @Override + public String toString() { + + if (this.nodeName != null) { + return nodeName; + } else { + return "[" + index + "]"; + } + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/sdk/EigenCalculateSDK.java b/arex-compare/src/main/java/io/arex/agent/compare/sdk/EigenCalculateSDK.java new file mode 100644 index 000000000..6e2ddf016 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/sdk/EigenCalculateSDK.java @@ -0,0 +1,23 @@ +package io.arex.agent.compare.sdk; + +import io.arex.agent.compare.eigen.EigenCalculateHandler; +import io.arex.agent.compare.model.eigen.EigenOptions; +import io.arex.agent.compare.model.RulesConfig; +import io.arex.agent.compare.model.eigen.EigenResult; +import io.arex.agent.compare.utils.EigenOptionsToRulesConvert; +import io.arex.agent.compare.utils.LogUtil; + +public class EigenCalculateSDK { + private static final EigenCalculateHandler eigenHandler = new EigenCalculateHandler(); + + public EigenResult calculateEigen(String msg, EigenOptions eigenOptions) { + EigenResult eigenResult = null; + try { + RulesConfig rulesConfig = EigenOptionsToRulesConvert.convert(msg, eigenOptions); + eigenResult = eigenHandler.doHandler(rulesConfig); + } catch (Throwable e) { + LogUtil.warn("calculate eigen value fail", e); + } + return eigenResult; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/EigenOptionsToRulesConvert.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/EigenOptionsToRulesConvert.java new file mode 100644 index 000000000..935818a6a --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/EigenOptionsToRulesConvert.java @@ -0,0 +1,33 @@ +package io.arex.agent.compare.utils; + +import io.arex.agent.compare.model.RulesConfig; +import io.arex.agent.compare.model.eigen.EigenOptions; + +import java.util.ArrayList; + +public class EigenOptionsToRulesConvert { + + public static RulesConfig convert(String msg, EigenOptions eigenOptions) { + RulesConfig rulesConfig = new RulesConfig(); + rulesConfig.setBaseMsg(msg); + rulesConfig.setNameToLower(true); + copyOptionsToRules(eigenOptions, rulesConfig); + configToLower(rulesConfig); + return rulesConfig; + } + + private static void copyOptionsToRules(EigenOptions eigenOptions, RulesConfig rulesConfig) { + if (eigenOptions == null) { + return; + } + rulesConfig.setCategoryType(eigenOptions.getCategoryType()); + rulesConfig.setExclusions(eigenOptions.getExclusions() == null ? null + : new ArrayList<>(eigenOptions.getExclusions())); + rulesConfig.setIgnoreNodeSet(eigenOptions.getIgnoreNodes()); + } + + private static void configToLower(RulesConfig rulesConfig) { + rulesConfig.setExclusions(FieldToLowerUtil.listListToLower(rulesConfig.getExclusions())); + rulesConfig.setIgnoreNodeSet(FieldToLowerUtil.setToLower(rulesConfig.getIgnoreNodeSet())); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/FieldToLowerUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/FieldToLowerUtil.java new file mode 100644 index 000000000..b71b0f1ce --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/FieldToLowerUtil.java @@ -0,0 +1,29 @@ +package io.arex.agent.compare.utils; + +import java.util.*; +import java.util.stream.Collectors; + +public class FieldToLowerUtil { + + public static List> listListToLower(List> lists) { + if (lists == null || lists.isEmpty()) { + return null; + } + List> result = new ArrayList<>(); + lists.forEach(item -> { + result.add(item.stream() + .map(String::toLowerCase) + .collect(Collectors.toList())); + }); + return result; + } + + public static Set setToLower(Collection collection) { + if (collection == null) { + return null; + } + return collection.stream().filter(Objects::nonNull) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/IgnoreUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/IgnoreUtil.java new file mode 100644 index 000000000..85632ea62 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/IgnoreUtil.java @@ -0,0 +1,62 @@ +package io.arex.agent.compare.utils; + +import io.arex.agent.compare.model.enumeration.Constant; + +import java.util.List; +import java.util.Set; + +/** + * Created by rchen9 on 2022/9/22. + */ +public class IgnoreUtil { + + public static boolean ignoreProcessor(List nodePath, List> ignoreNodePaths, + Set ignoreNodeSet) { + if (ignoreNodeProcessor(nodePath, ignoreNodeSet)) { + return true; + } + + if (ignoreNodePaths != null && !ignoreNodePaths.isEmpty()) { + for (List ignoreNodePath : ignoreNodePaths) { + if (ignoreMatch(nodePath, ignoreNodePath)) { + return true; + } + } + } + return false; + } + + private static boolean ignoreMatch(List pathInList, List ignoreNodePath) { + + int size = ignoreNodePath.size(); + if (size > pathInList.size()) { + return false; + } + + for (int i = 0; i < size; i++) { + if (!ignoreNodePath.get(i).equals(pathInList.get(i)) && !ignoreNodePath.get(i) + .equals(Constant.DYNAMIC_PATH)) { + return false; + } + } + return true; + } + + private static boolean ignoreNodeProcessor(List nodePath, Set ignoreNodeSet) { + + if (ignoreNodeSet == null || ignoreNodeSet.isEmpty()) { + return false; + } + + if (nodePath == null || nodePath.isEmpty()) { + return false; + } + + for (String nodeName : nodePath) { + if (ignoreNodeSet.contains(nodeName)) { + return true; + } + } + return false; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/JacksonHelperUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/JacksonHelperUtil.java new file mode 100644 index 000000000..70196e7c6 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/JacksonHelperUtil.java @@ -0,0 +1,32 @@ +package io.arex.agent.compare.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class JacksonHelperUtil { + + public static ObjectMapper objectMapper = new ObjectMapper(); + + public static List getNames(ObjectNode objectNode) { + List result = new ArrayList<>(); + Iterator stringIterator = objectNode.fieldNames(); + while (stringIterator.hasNext()) { + result.add(stringIterator.next()); + } + return result; + } + + public static ObjectNode getObjectNode() { + return objectMapper.createObjectNode(); + } + + public static ArrayNode getArrayNode() { + return objectMapper.createArrayNode(); + } + +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/ListUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/ListUtil.java new file mode 100644 index 000000000..ff3e0a0da --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/ListUtil.java @@ -0,0 +1,14 @@ +package io.arex.agent.compare.utils; + +import java.util.List; + +public class ListUtil { + + public static void removeLast(List list) { + if (list == null || list.size() == 0) { + return; + } + list.remove(list.size() - 1); + } + +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/LogUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/LogUtil.java new file mode 100644 index 000000000..a664b88a9 --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/LogUtil.java @@ -0,0 +1,38 @@ +package io.arex.agent.compare.utils; + +import io.arex.agent.bootstrap.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LogUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(LogUtil.class); + + public static String buildTitle(String title) { + return StringUtil.format("[[title=arex.%s]]", title); + } + + public static void info(String title, String message) { + String logMessage = buildMessage(buildTitle(title), message); + LOGGER.info(logMessage); + } + + public static void warn(String title, Throwable ex) { + warn(title, null, ex); + } + + public static void warn(String title, String message) { + warn(title, message, null); + } + + public static void warn(String title, String message, Throwable exception) { + String logMessage = buildMessage(buildTitle(title), message); + LOGGER.warn(logMessage, exception); + } + + private static String buildMessage(String title, String message) { + if (StringUtil.isEmpty(message)) { + return title; + } + return title + message; + } +} diff --git a/arex-compare/src/main/java/io/arex/agent/compare/utils/NameConvertUtil.java b/arex-compare/src/main/java/io/arex/agent/compare/utils/NameConvertUtil.java new file mode 100644 index 000000000..ff0a26edb --- /dev/null +++ b/arex-compare/src/main/java/io/arex/agent/compare/utils/NameConvertUtil.java @@ -0,0 +1,47 @@ +package io.arex.agent.compare.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.List; + +public class NameConvertUtil { + + public static void nameConvert(Object object) { + if (object == null || object instanceof NullNode) { + return; + } + + if (object instanceof ObjectNode) { + ObjectNode jsonObj1 = (ObjectNode) object; + List names = JacksonHelperUtil.getNames(jsonObj1); + + for (String fieldName : names) { + JsonNode obj1FieldValue = jsonObj1.get(fieldName); + jsonObj1.set(fieldName.toLowerCase(), obj1FieldValue); + nameConvert(obj1FieldValue); + } + for (String fieldName : names) { + if (containsUpper(fieldName)) { + jsonObj1.remove(fieldName); + } + } + } else if (object instanceof ArrayNode) { + ArrayNode obj1Array = (ArrayNode) object; + int len = obj1Array.size(); + for (int i = 0; i < len; i++) { + Object element = obj1Array.get(i); + nameConvert(element); + } + } + + } + + public static boolean containsUpper(String name) { + return name.chars().anyMatch( + (int ch) -> Character.isUpperCase((char) ch) + ); + } +} diff --git a/arex-instrumentation-api/pom.xml b/arex-instrumentation-api/pom.xml index ce6ae7772..2bae1d8ad 100644 --- a/arex-instrumentation-api/pom.xml +++ b/arex-instrumentation-api/pom.xml @@ -16,6 +16,10 @@ ${project.groupId} arex-agent-bootstrap + + ${project.groupId} + arex-compare + net.bytebuddy byte-buddy diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/Config.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/Config.java index 7fea9f3c8..70b924f6e 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/Config.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/Config.java @@ -5,6 +5,7 @@ import io.arex.agent.bootstrap.util.StringUtil; import io.arex.inst.runtime.context.RecordLimiter; import io.arex.inst.runtime.listener.EventProcessor; +import io.arex.inst.runtime.model.CompareConfigurationEntity; import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.DynamicClassEntity; @@ -23,12 +24,10 @@ public class Config { private static final char SEPARATOR = ','; private static Config INSTANCE = null; - static void update(boolean enableDebug, String serviceName, List dynamicClassList, - Map properties, Set excludeServiceOperations, - int dubboStreamReplayThreshold, int recordRate) { - INSTANCE = new Config(enableDebug, serviceName, dynamicClassList, properties, - excludeServiceOperations, - dubboStreamReplayThreshold, recordRate); + static void update(boolean enableDebug, String serviceName, + int dubboStreamReplayThreshold, int recordRate, ConfigExtendEntity extendEntity) { + INSTANCE = new Config(enableDebug, serviceName, + dubboStreamReplayThreshold, recordRate, extendEntity); } public static Config get() { @@ -48,15 +47,15 @@ public static Config get() { private final Set includeServiceOperations; private final Set coveragePackages = new ConcurrentHashSet<>(); private final Map mockerTags; + private final CompareConfigurationEntity compareConfigurationEntity; - Config(boolean enableDebug, String serviceName, List dynamicClassList, - Map properties, - Set excludeServiceOperations, int dubboStreamReplayThreshold, int recordRate) { + Config(boolean enableDebug, String serviceName, int dubboStreamReplayThreshold, + int recordRate, ConfigExtendEntity extendEntity) { this.enableDebug = enableDebug; this.serviceName = serviceName; - this.dynamicClassList = dynamicClassList; - this.properties = properties; - this.excludeServiceOperations = buildExcludeServiceOperations(excludeServiceOperations); + this.dynamicClassList = extendEntity.getDynamicClassList(); + this.properties = extendEntity.getProperties(); + this.excludeServiceOperations = buildExcludeServiceOperations(extendEntity.getExcludeServiceOperations()); this.dubboStreamReplayThreshold = dubboStreamReplayThreshold; this.recordRate = recordRate; this.recordVersion = properties.get("arex.agent.version"); @@ -64,6 +63,7 @@ public static Config get() { this.mockerTags = StringUtil.asMap(System.getProperty(ConfigConstants.MOCKER_TAGS)); buildCoveragePackages(properties); buildDynamicClassInfo(); + this.compareConfigurationEntity = extendEntity.getCompareConfigurationEntity(); } /** @@ -210,6 +210,10 @@ public boolean isLocalStorage() { return STORAGE_MODE.equalsIgnoreCase(getString(STORAGE_SERVICE_MODE)); } + public CompareConfigurationEntity getCompareConfiguration() { + return compareConfigurationEntity; + } + /** * Conditions for determining invalid recording configuration:
* 1. rate <= 0
diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigBuilder.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigBuilder.java index edcfbfa53..d8925fe90 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigBuilder.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigBuilder.java @@ -1,5 +1,6 @@ package io.arex.inst.runtime.config; +import io.arex.inst.runtime.model.CompareConfigurationEntity; import io.arex.inst.runtime.model.DynamicClassEntity; import java.util.*; @@ -12,6 +13,7 @@ public class ConfigBuilder { private Set excludeServiceOperations; private int dubboStreamReplayThreshold; private int recordRate; + private CompareConfigurationEntity compareConfigurationEntity; public static ConfigBuilder create(String serviceName) { return new ConfigBuilder(serviceName); @@ -37,6 +39,11 @@ public ConfigBuilder excludeServiceOperations(Set excludeServiceOperatio return this; } + public ConfigBuilder compareConfiguration(CompareConfigurationEntity compareConfigurationEntity) { + this.compareConfigurationEntity = compareConfigurationEntity; + return this; + } + public ConfigBuilder addProperty(String name, String value) { if (value != null) { properties.put(name, value); @@ -70,7 +77,8 @@ public ConfigBuilder recordRate(int recordRate) { } public void build() { - Config.update(enableDebug, serviceName, dynamicClassList, Collections.unmodifiableMap(new HashMap<>(properties)), - excludeServiceOperations, dubboStreamReplayThreshold, recordRate); + Config.update(enableDebug, serviceName, dubboStreamReplayThreshold, recordRate, + ConfigExtendEntity.of(dynamicClassList, Collections.unmodifiableMap(new HashMap<>(properties)), + excludeServiceOperations, compareConfigurationEntity)); } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigExtendEntity.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigExtendEntity.java new file mode 100644 index 000000000..609268606 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/config/ConfigExtendEntity.java @@ -0,0 +1,47 @@ +package io.arex.inst.runtime.config; + +import io.arex.inst.runtime.model.CompareConfigurationEntity; +import io.arex.inst.runtime.model.DynamicClassEntity; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ConfigExtendEntity { + private final List dynamicClassList; + private final Map properties; + private final Set excludeServiceOperations; + private final CompareConfigurationEntity compareConfigurationEntity; + + private ConfigExtendEntity(List dynamicClassList, + Map properties, Set excludeServiceOperations, + CompareConfigurationEntity compareConfigurationEntity) { + this.dynamicClassList = dynamicClassList; + this.properties = properties; + this.excludeServiceOperations = excludeServiceOperations; + this.compareConfigurationEntity = compareConfigurationEntity; + } + + public static ConfigExtendEntity of(List dynamicClassList, + Map properties, Set excludeServiceOperations, + CompareConfigurationEntity compareConfigurationEntity) { + return new ConfigExtendEntity(dynamicClassList, properties, excludeServiceOperations, + compareConfigurationEntity); + } + + public List getDynamicClassList() { + return dynamicClassList; + } + + public Map getProperties() { + return properties; + } + + public Set getExcludeServiceOperations() { + return excludeServiceOperations; + } + + public CompareConfigurationEntity getCompareConfigurationEntity() { + return compareConfigurationEntity; + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ArexContext.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ArexContext.java index bbc28b75c..9b26fab1d 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ArexContext.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ArexContext.java @@ -4,7 +4,8 @@ import io.arex.agent.bootstrap.util.ConcurrentHashSet; import io.arex.agent.bootstrap.util.StringUtil; import io.arex.inst.runtime.model.ArexConstants; -import io.arex.inst.runtime.util.MergeRecordUtil; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; +import io.arex.inst.runtime.util.MockManager; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -31,6 +32,8 @@ public class ArexContext { private static final AtomicIntegerFieldUpdater SEQUENCE_UPDATER = AtomicIntegerFieldUpdater.newUpdater(ArexContext.class, "sequence"); + private LinkedBlockingQueue replayCompareResultQueue; + public static ArexContext of(String caseId) { return of(caseId, null); } @@ -161,11 +164,19 @@ public LinkedBlockingQueue getMergeRecordQueue() { return mergeRecordQueue; } + public LinkedBlockingQueue getReplayCompareResultQueue() { + if (replayCompareResultQueue == null) { + replayCompareResultQueue = new LinkedBlockingQueue<>(); + } + return replayCompareResultQueue; + } + public void clear() { if (methodSignatureHashList != null) { methodSignatureHashList.clear(); } if (cachedReplayResultMap != null) { + MockManager.saveReplayRemainCompareRelation(this); cachedReplayResultMap.clear(); } if (excludeMockTemplate != null) { @@ -176,8 +187,11 @@ public void clear() { } if (mergeRecordQueue != null) { // async thread merge record (main entry has ended) - MergeRecordUtil.recordRemain(this); + MockManager.recordRemain(this); mergeRecordQueue.clear(); } + if (replayCompareResultQueue != null) { + replayCompareResultQueue.clear(); + } } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ContextManager.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ContextManager.java index 4f2b4dd1c..e42aaaf89 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ContextManager.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/ContextManager.java @@ -100,4 +100,8 @@ public static void setAttachment(String key, Object value) { context.setAttachment(key, value); } } + + public static void clear() { + RECORD_MAP.clear(); + } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/LatencyContextHashMap.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/LatencyContextHashMap.java index 811f19331..7015a0e53 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/LatencyContextHashMap.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/context/LatencyContextHashMap.java @@ -34,6 +34,12 @@ public ArexContext remove(Object key) { return super.get(key); } + @Override + public void clear() { + overdueCleanUp(); + super.clear(); + } + private void overdueCleanUp() { if (CLEANUP_LOCK.tryLock()) { try { diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/listener/EventProcessor.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/listener/EventProcessor.java index e2d1b3556..9156c669e 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/listener/EventProcessor.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/listener/EventProcessor.java @@ -9,6 +9,7 @@ import io.arex.agent.bootstrap.util.StringUtil; import io.arex.agent.bootstrap.util.ServiceLoader; import io.arex.inst.runtime.config.Config; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.InitializeEnum; import io.arex.inst.runtime.request.RequestHandlerManager; import io.arex.inst.runtime.log.LogManager; @@ -24,6 +25,8 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import io.arex.inst.runtime.util.ReplayUtil; @@ -37,6 +40,10 @@ public class EventProcessor { private static final AtomicReference INIT_DEPENDENCY = new AtomicReference<>(InitializeEnum.START); private static final Method FIND_LOADED_METHOD = ReflectUtil.getMethod(ClassLoader.class, "findLoadedClass", String.class); private static boolean existJacksonDependency = true; + + private static final ScheduledThreadPoolExecutor SCHEDULER = + new ScheduledThreadPoolExecutor(1, r -> new Thread(r, "arex-replay-manager-thread")); + static { try { Class.forName("com.fasterxml.jackson.databind.ObjectMapper",true, Thread.currentThread().getContextClassLoader()); @@ -53,7 +60,7 @@ public static void onCreate(EventSource source){ return; } initContext(source); - loadReplayData(); + initReplay(); initClock(); addEnterLog(); } @@ -111,7 +118,15 @@ private static void initClock(){ } public static void onExit(){ + ReplayUtil.saveReplayCompareResult(); ContextManager.remove(); + // if replay plan complete end(the last replay case), delay clear context (include async thread context) + ArexContext context = ContextManager.currentContext(); + if (context.getAttachment(ArexConstants.REPLAY_END_FLAG) != null + && Boolean.parseBoolean(String.valueOf(context.getAttachment(ArexConstants.REPLAY_END_FLAG)))) { + // must contain LatencyContextHashMap#overdueCleanUp time(1 minutes) + SCHEDULER.schedule(ContextManager::clear, 2, TimeUnit.MINUTES); + } } /** @@ -177,7 +192,8 @@ public static boolean dependencyInitComplete() { return InitializeEnum.COMPLETE.equals(INIT_DEPENDENCY.get()); } - private static void loadReplayData() { + private static void initReplay() { + // init replay and cached all mockers within case ReplayUtil.queryMockers(); } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/log/LogManager.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/log/LogManager.java index 635edca06..86968b2a1 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/log/LogManager.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/log/LogManager.java @@ -92,4 +92,17 @@ private static boolean useExtensionLog() { public static void setContextMap(Map contextMap) { MDC.setContextMap(Objects.isNull(contextMap) ? new HashMap<>() : contextMap); } + + public static void info(ArexContext currentContext, String title, String message) { + String logMessage = buildMessage(buildTitle(title), message); + if (useExtensionLog() && currentContext != null) { + for (Logger extensionLogger : EXTENSION_LOGGER_LIST) { + extensionLogger.addTag(currentContext.getCaseId(), currentContext.getReplayId()); + extensionLogger.info(logMessage); + } + return; + } + + LOGGER.info(logMessage); + } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/EigenMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/EigenMatchStrategy.java deleted file mode 100644 index 2f6a08e3a..000000000 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/EigenMatchStrategy.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.arex.inst.runtime.match; - -import io.arex.inst.runtime.model.MatchStrategyEnum; - -public class EigenMatchStrategy extends AbstractMatchStrategy{ - - /** - * search by eigen value of request - */ - void process(MatchStrategyContext context) { - context.setMatchStrategy(MatchStrategyEnum.EIGEN); - // to be implemented after database merge replay support - } -} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/FuzzyMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/FuzzyMatchStrategy.java deleted file mode 100644 index 6478c3b20..000000000 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/FuzzyMatchStrategy.java +++ /dev/null @@ -1,44 +0,0 @@ -package io.arex.inst.runtime.match; - -import io.arex.agent.bootstrap.model.MockStrategyEnum; -import io.arex.agent.bootstrap.model.Mocker; -import io.arex.agent.bootstrap.util.CollectionUtil; -import io.arex.inst.runtime.model.MatchStrategyEnum; - -import java.util.List; - -public class FuzzyMatchStrategy extends AbstractMatchStrategy { - /** - * search under the same method signature - * replayList is arranged in ascending order by creationTime - * @return not matched before or last one - */ - void process(MatchStrategyContext context) { - context.setMatchStrategy(MatchStrategyEnum.FUZZY); - List replayList = context.getReplayList(); - Mocker mocker = null; - int size = replayList.size(); - for (int i = 0; i < size; i++) { - Mocker mockerDTO = replayList.get(i); - if (!mockerDTO.isMatched()) { - mocker = mockerDTO; - break; - } - } - if (mocker == null && MockStrategyEnum.FIND_LAST == context.getMockStrategy()) { - mocker = replayList.get(size - 1); - } - if (mocker != null) { - mocker.setMatched(true); - } else { - context.setReason("fuzzy match no result, all has been matched"); - } - - context.setMatchMocker(mocker); - } - - @Override - boolean internalCheck(MatchStrategyContext context) { - return CollectionUtil.isNotEmpty(context.getReplayList()); - } -} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchKeyFactory.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchKeyFactory.java new file mode 100644 index 000000000..fd796bbe8 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchKeyFactory.java @@ -0,0 +1,62 @@ +package io.arex.inst.runtime.match; + +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.inst.runtime.match.key.DatabaseMatchKeyBuilderImpl; +import io.arex.inst.runtime.match.key.DefaultMatchKeyBuilderImpl; +import io.arex.inst.runtime.match.key.HttpClientMatchKeyBuilderImpl; +import io.arex.inst.runtime.match.key.MatchKeyBuilder; + +import java.util.ArrayList; +import java.util.List; + +public class MatchKeyFactory { + public static final MatchKeyFactory INSTANCE = new MatchKeyFactory(); + + private final List matchKeyBuilders; + + private MatchKeyFactory() { + this.matchKeyBuilders = new ArrayList<>(); + matchKeyBuilders.add(new HttpClientMatchKeyBuilderImpl()); + matchKeyBuilders.add(new DatabaseMatchKeyBuilderImpl()); + + // DefaultMatchKeyBuilderImpl must be the last one + matchKeyBuilders.add(new DefaultMatchKeyBuilderImpl()); + } + + private MatchKeyBuilder find(MockCategoryType categoryType) { + if (CollectionUtil.isNotEmpty(this.matchKeyBuilders)) { + for (MatchKeyBuilder matchKeyBuilder : this.matchKeyBuilders) { + if (matchKeyBuilder.isSupported(categoryType)) { + return matchKeyBuilder; + } + } + } + return null; + } + + public int getFuzzyMatchKey(Mocker mocker) { + MatchKeyBuilder matchKeyBuilder = find(mocker.getCategoryType()); + if (matchKeyBuilder == null) { + return 0; + } + return matchKeyBuilder.getFuzzyMatchKey(mocker); + } + + public int getAccurateMatchKey(Mocker mocker) { + MatchKeyBuilder matchKeyBuilder = find(mocker.getCategoryType()); + if (matchKeyBuilder == null) { + return 0; + } + return matchKeyBuilder.getAccurateMatchKey(mocker); + } + + public String getEigenBody(Mocker mocker) { + MatchKeyBuilder matchKeyBuilder = find(mocker.getCategoryType()); + if (matchKeyBuilder == null) { + return null; + } + return matchKeyBuilder.getEigenBody(mocker); + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyContext.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyContext.java index 2e94bc726..e8a4f1f0a 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyContext.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyContext.java @@ -8,16 +8,15 @@ public class MatchStrategyContext { private Mocker requestMocker; - private List replayList; + private List recordList; private MockStrategyEnum mockStrategy; private boolean interrupt; private Mocker matchMocker; private MatchStrategyEnum matchStrategy; private String reason; - public MatchStrategyContext(Mocker requestMocker, List replayList, MockStrategyEnum mockStrategy) { + public MatchStrategyContext(Mocker requestMocker, MockStrategyEnum mockStrategy) { this.requestMocker = requestMocker; - this.replayList = replayList; this.mockStrategy = mockStrategy; } @@ -29,12 +28,12 @@ public void setRequestMocker(Mocker requestMocker) { this.requestMocker = requestMocker; } - public List getReplayList() { - return replayList; + public List getRecordList() { + return recordList; } - public void setReplayList(List replayList) { - this.replayList = replayList; + public void setRecordList(List recordList) { + this.recordList = recordList; } public MockStrategyEnum getMockStrategy() { diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyRegister.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyRegister.java index aa77ebd70..062adee8a 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyRegister.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/MatchStrategyRegister.java @@ -1,7 +1,12 @@ package io.arex.inst.runtime.match; import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.inst.runtime.match.strategy.AbstractMatchStrategy; +import io.arex.inst.runtime.match.strategy.AccurateMatchStrategy; +import io.arex.inst.runtime.match.strategy.EigenMatchStrategy; +import io.arex.inst.runtime.match.strategy.FuzzyMatchStrategy; import java.util.*; @@ -9,20 +14,32 @@ public class MatchStrategyRegister { private static final AbstractMatchStrategy ACCURATE_STRATEGY = new AccurateMatchStrategy(); private static final AbstractMatchStrategy FUZZY_STRATEGY = new FuzzyMatchStrategy(); private static final AbstractMatchStrategy EIGEN_STRATEGY = new EigenMatchStrategy(); + + private static final List ACCURATE_FUZZY_COMBINE = CollectionUtil.newArrayList(ACCURATE_STRATEGY, FUZZY_STRATEGY); + + private static final List DEFAULT_MATCH_COMBINE = CollectionUtil.newArrayList(ACCURATE_STRATEGY, EIGEN_STRATEGY); + private static final Map> MATCH_STRATEGIES = register(); private MatchStrategyRegister() { } - public static List getMatchStrategies(MockCategoryType categoryType) { - return MATCH_STRATEGIES.get(categoryType.getName()); + public static List getMatchStrategies(Mocker mocker, int fuzzyMatchResultCount) { + if (fuzzyMatchResultCount == 1) { + return ACCURATE_FUZZY_COMBINE; + } + List matchStrategies = MATCH_STRATEGIES.get(mocker.getCategoryType().getName()); + if (matchStrategies == null) { + return DEFAULT_MATCH_COMBINE; + } + return matchStrategies; } private static Map> register() { Map> strategyMap = new HashMap<>(); - strategyMap.put(MockCategoryType.DYNAMIC_CLASS.getName(), CollectionUtil.newArrayList(ACCURATE_STRATEGY, FUZZY_STRATEGY)); - strategyMap.put(MockCategoryType.REDIS.getName(), CollectionUtil.newArrayList(ACCURATE_STRATEGY, FUZZY_STRATEGY)); - strategyMap.put(MockCategoryType.DATABASE.getName(), CollectionUtil.newArrayList(ACCURATE_STRATEGY, EIGEN_STRATEGY)); + strategyMap.put(MockCategoryType.DYNAMIC_CLASS.getName(), ACCURATE_FUZZY_COMBINE); + strategyMap.put(MockCategoryType.REDIS.getName(), ACCURATE_FUZZY_COMBINE); + // other category type such as:database、httpclient use default match strategy: accurate and eigen match return strategyMap; } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/ReplayMatcher.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/ReplayMatcher.java index b5f331526..7d735b6b8 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/ReplayMatcher.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/ReplayMatcher.java @@ -1,20 +1,25 @@ package io.arex.inst.runtime.match; +import io.arex.agent.bootstrap.model.MockCategoryType; import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.agent.bootstrap.util.MapUtils; import io.arex.agent.bootstrap.util.StringUtil; import io.arex.inst.runtime.config.Config; import io.arex.inst.runtime.context.ContextManager; import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.match.strategy.AbstractMatchStrategy; +import io.arex.inst.runtime.model.ArexConstants; +import io.arex.inst.runtime.model.MatchStrategyEnum; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; -import io.arex.inst.runtime.util.MockUtils; +import io.arex.inst.runtime.util.ReplayUtil; import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; public class ReplayMatcher { - private static final String MATCH_TITLE = "replay.match"; - private ReplayMatcher() { } @@ -25,41 +30,81 @@ private ReplayMatcher() { * 3. fuzzy match/eigen match */ public static Mocker match(Mocker requestMocker, MockStrategyEnum mockStrategy) { - Map> cachedReplayResultMap = ContextManager.currentContext().getCachedReplayResultMap(); - // first match methodRequestTypeHash: category + operationName + requestType, ensure the same method - List replayList = cachedReplayResultMap.get(MockUtils.methodRequestTypeHash(requestMocker)); - if (CollectionUtil.isEmpty(replayList)) { - LogManager.warn(MATCH_TITLE, StringUtil.format("match no result, not exist this method signature, " + - "check if it has been recorded, categoryType: %s, operationName: %s, requestBody: %s", - requestMocker.getCategoryType().getName(), requestMocker.getOperationName(), requestMocker.getTargetRequest().getBody())); + if (MapUtils.isEmpty(ContextManager.currentContext().getCachedReplayResultMap())) { return null; } - List matchStrategyList = MatchStrategyRegister.getMatchStrategies(requestMocker.getCategoryType()); - MatchStrategyContext context = new MatchStrategyContext(requestMocker, replayList, mockStrategy); - for (AbstractMatchStrategy matchStrategy : matchStrategyList) { - matchStrategy.match(context); - } + MatchStrategyContext context = new MatchStrategyContext(requestMocker, mockStrategy); + + doMatch(context); logMatchResult(context); + saveCompareResult(context); + return context.getMatchMocker(); } + private static void doMatch(MatchStrategyContext context) { + Mocker requestMocker = context.getRequestMocker(); + Map> cachedReplayResultMap = ContextManager.currentContext().getCachedReplayResultMap(); + // pre match for all mocker category type + List recordList = preMatch(context, requestMocker, cachedReplayResultMap); + if (CollectionUtil.isEmpty(recordList)) { + return; + } + context.setRecordList(recordList); + int fuzzyMatchResultCount = recordList.size(); + List matchStrategyList = MatchStrategyRegister.getMatchStrategies(requestMocker, fuzzyMatchResultCount); + // multi thread match safe + synchronized (cachedReplayResultMap) { + for (AbstractMatchStrategy matchStrategy : matchStrategyList) { + matchStrategy.match(context); + } + } + } + + private static List preMatch(MatchStrategyContext context, Mocker requestMocker, Map> cachedReplayResultMap) { + // first match, such as: category + operationName + requestType, ensure the same method + requestMocker.setFuzzyMatchKey(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(requestMocker)); + List recordList = cachedReplayResultMap.get(requestMocker.getFuzzyMatchKey()); + recordList = compatibleNoRequestType(recordList, cachedReplayResultMap, requestMocker); + if (CollectionUtil.isEmpty(recordList)) { + context.setReason("match no result, not exist this method signature, check if it has been recorded or request type is empty"); + } + return recordList; + } + + private static List compatibleNoRequestType(List recordList, Map> cachedReplayResultMap, Mocker requestMocker) { + if (CollectionUtil.isEmpty(recordList)) { + String categoryType = requestMocker.getCategoryType().getName(); + if (MockCategoryType.DYNAMIC_CLASS.getName().equals(categoryType) || MockCategoryType.REDIS.getName().equals(categoryType)) { + // dynamic class or redis may not record requestType on old version + requestMocker.getTargetRequest().setType(null); + requestMocker.setFuzzyMatchKey(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(requestMocker)); + return cachedReplayResultMap.get(requestMocker.getFuzzyMatchKey()); + } + } + return recordList; + } + private static void logMatchResult(MatchStrategyContext context) { Mocker matchedMocker = context.getMatchMocker(); Mocker requestMocker = context.getRequestMocker(); - StringBuilder matchResult = new StringBuilder(requestMocker.getCategoryType().getName()); - matchResult.append(":").append(requestMocker.getOperationName()); + if (requestMocker.getCategoryType().isEntryPoint()) { + return; + } + + String matchResult; if (matchedMocker != null) { - matchResult.append(" match success"); + matchResult = "match success"; } else { - matchResult.append(" match fail, reason: ").append(context.getReason()); + matchResult = "match fail" + StringUtil.format(", reason: %s", context.getReason()); } String message = StringUtil.format("%s %n%s, requestType: %s, match strategy: %s, mock strategy: %s", - matchResult.toString(), + matchResult, requestMocker.logBuilder().toString(), requestMocker.getTargetRequest().getType(), (context.getMatchStrategy() != null ? context.getMatchStrategy().name() : StringUtil.EMPTY), @@ -69,6 +114,47 @@ private static void logMatchResult(MatchStrategyContext context) { message += StringUtil.format("%nrequest: %s%nresponse: %s", Serializer.serialize(requestMocker), Serializer.serialize(matchedMocker)); } - LogManager.info(MATCH_TITLE, message); + LogManager.info(ArexConstants.MATCH_LOG_TITLE, message); + } + + /** + * compare type: + * value diff + * new call (recordMocker is null) + * (call missing after entry point) + */ + private static boolean saveCompareResult(MatchStrategyContext context) { + Mocker replayMocker = context.getRequestMocker(); + boolean isEntryPoint = replayMocker.getCategoryType().isEntryPoint(); + if (replayMocker.getCategoryType().isSkipComparison() && !isEntryPoint) { + return false; + } + + String recordMsg = null; + String replayMsg = ReplayUtil.getCompareMessage(replayMocker); + long recordTime = Long.MAX_VALUE; + long replayTime = replayMocker.getCreationTime(); + boolean sameMsg = false; + Mocker recordMocker = context.getMatchMocker(); + String extendMessage = null; + if (recordMocker != null) { + recordMsg = ReplayUtil.getCompareMessage(recordMocker); + recordTime = recordMocker.getCreationTime(); + if (MatchStrategyEnum.ACCURATE.equals(context.getMatchStrategy()) && !isEntryPoint) { + replayMsg = StringUtil.EMPTY; + sameMsg = true; + } + // for expect-script assert use + if ("SOAConsumer".equalsIgnoreCase(recordMocker.getCategoryType().getName())) { + extendMessage = recordMocker.getTargetResponse().getBody(); + } + } + + LinkedBlockingQueue replayCompareResultQueue = + ContextManager.currentContext().getReplayCompareResultQueue(); + ReplayCompareResultDTO compareResultDTO = ReplayUtil.convertCompareResult( + replayMocker, recordMsg, replayMsg, recordTime, replayTime, sameMsg); + compareResultDTO.setExtendMessage(extendMessage); + return replayCompareResultQueue.offer(compareResultDTO); } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImpl.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImpl.java new file mode 100644 index 000000000..97e212bbd --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImpl.java @@ -0,0 +1,186 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.inst.runtime.model.ArexConstants; +import io.arex.inst.runtime.serializer.Serializer; +import io.arex.inst.runtime.util.DatabaseUtils; + +import java.util.*; + +public class DatabaseMatchKeyBuilderImpl implements MatchKeyBuilder { + + private static final char SQL_BATCH_TERMINAL_CHAR = ';'; + private static final int INDEX_NOT_FOUND = -1; + private static final int UPPER_LOWER_CASE_DELTA_VALUE = 32; + /** + * table name for ms-sql-server and mysql, which valid format as follow: ms-sql-server example: + * 1,dbo.[tableName] 2,[orderDb].dbo.[tableName] 3,tableName mysql example: + * 1,`orderDb`.`tableName' 2,`tableName' 3, tableName + *

+ * table name for inner join as short,as follow: SELECT * FROM db.`tableNameA` a, tableNameB` b + * WHERE a.id = b.id + *

+ * for example: SELECT * FROM tableNameA a INNER JOIN `tableNameB` b ON a.id = b.id; SELECT * FROM + * tableNameA a LEFT JOIN db.tableNameB b ON a.id = b.id; SELECT * FROM tableNameA a RIGHT JOIN + * tableNameB b ON a.id = b.id; + */ + private static final List SQL_TABLE_KEYS = Arrays.asList("from", "join", "update", "into"); + + @Override + public boolean isSupported(MockCategoryType categoryType) { + return MockCategoryType.DATABASE.getName().equals(categoryType.getName()); + } + + /** + * category + dbName + tableName + operationName + */ + @Override + public int getFuzzyMatchKey(Mocker mocker) { + String operationName = mocker.getOperationName(); + Mocker.Target request = mocker.getTargetRequest(); + String dbName = DatabaseUtils.parseDbName(operationName, request.attributeAsString(ArexConstants.DB_NAME)); + return StringUtil.encodeAndHash( + mocker.getCategoryType().getName(), + dbName, + getTableName(operationName, request.getBody()), + operationName); + } + + + /** + * sql + parameters + */ + @Override + public int getAccurateMatchKey(Mocker mocker) { + Mocker.Target request = mocker.getTargetRequest(); + String sql = request.getBody(); + String parameters = request.attributeAsString(ArexConstants.DB_PARAMETERS); + return StringUtil.encodeAndHash( + sql, + parameters); + } + + /** + * sql + parameters + */ + @Override + public String getEigenBody(Mocker mocker) { + String parameters = mocker.getTargetRequest().attributeAsString(ArexConstants.DB_PARAMETERS); + Map objectNode = new HashMap<>(); + objectNode.put(ArexConstants.DB_SQL, mocker.getTargetRequest().getBody()); + if (StringUtil.isNotEmpty(parameters)) { + objectNode.put(ArexConstants.DB_PARAMETERS, parameters); + } + return Serializer.serialize(objectNode); + } + + private String getTableName(String operationName, String sqlText) { + List tableNames = DatabaseUtils.parseTableNames(operationName); + if (CollectionUtil.isNotEmpty(tableNames)) { + return String.join(",", tableNames); + } + return findTableName(sqlText); + } + + private String findTableName(String sqlText) { + int sourceCount = sqlText.length(); + if (sourceCount > ArexConstants.DB_SQL_MAX_LEN) { + return StringUtil.EMPTY; + } + List tableNames = new ArrayList<>(); + for (int i = 0; i < SQL_TABLE_KEYS.size(); i++) { + String key = SQL_TABLE_KEYS.get(i); + int targetCount = key.length(); + int fromIndex = 0; + int index = findIndexWholeIgnoreCase(sqlText, sourceCount, key, targetCount, fromIndex); + while (index != INDEX_NOT_FOUND) { + fromIndex = index + targetCount; + int skipWhitespaceCount = skipWhitespace(sqlText, fromIndex, sourceCount); + fromIndex += skipWhitespaceCount; + String tableName = readTableValue(sqlText, fromIndex, sourceCount); + int tableNameLength = tableName.length(); + fromIndex += tableNameLength; + tableNames.add(tableName); + index = findIndexWholeIgnoreCase(sqlText, sourceCount, key, targetCount, fromIndex); + } + } + return String.join(",", tableNames); + } + + private static String readTableValue(String sqlText, int readFromIndex, int sourceCount) { + final int valueBeginIndex = readFromIndex; + for (; readFromIndex < sourceCount; readFromIndex++) { + if (readShouldTerminal(sqlText.charAt(readFromIndex))) { + break; + } + } + return sqlText.substring(valueBeginIndex, readFromIndex); + } + + private static int findIndexWholeIgnoreCase(String source, int sourceCount, String target, + int targetCount, + int fromIndex) { + if (fromIndex >= sourceCount) { + return INDEX_NOT_FOUND; + } + char first = target.charAt(0); + int max = sourceCount - targetCount; + for (int i = fromIndex; i <= max; i++) { + if (firstCharacterWordBoundaryNotMatch(source, first, i)) { + while (++i <= max && firstCharacterWordBoundaryNotMatch(source, first, i)) {} + } + // Found first character, now look at the rest of target + if (findFirstChar(i, max, targetCount, source, target)) { + return i; + } + } + return INDEX_NOT_FOUND; + } + + private static boolean findFirstChar(int i, int max, int targetCount, String source, String target) { + if (i <= max) { + int j = i + 1; + int end = j + targetCount - 1; + for (int k = 1; j < end && equalsIgnoreCase(source.charAt(j), target.charAt(k)); j++, k++) {} + if (j == end && isWordBoundary(source, j)) { + // Found whole string + return true; + } + } + return false; + } + + private static boolean readShouldTerminal(char src) { + return src == SQL_BATCH_TERMINAL_CHAR || isWhitespace(src); + } + + private static int skipWhitespace(String sqlText, int fromIndex, int sourceCount) { + int skipWhitespaceCount = 0; + for (; fromIndex < sourceCount && isWhitespace(sqlText.charAt(fromIndex)); + fromIndex++, skipWhitespaceCount++) {} + return skipWhitespaceCount; + } + + private static boolean isWhitespace(char src) { + return Character.isWhitespace(src); + } + + private static boolean firstCharacterWordBoundaryNotMatch(final String source, char first, int position) { + return !(equalsIgnoreCase(source.charAt(position), first) && isWordBoundary(source, + position - 1)); + } + + private static boolean isWordBoundary(final String source, int positionOfPrevOrNext) { + if (positionOfPrevOrNext < 0 || positionOfPrevOrNext >= source.length()) { + return true; + } + return isWhitespace(source.charAt(positionOfPrevOrNext)); + } + + private static boolean equalsIgnoreCase(char src, char target) { + return src == target || Math.abs(src - target) == UPPER_LOWER_CASE_DELTA_VALUE; + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImpl.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImpl.java new file mode 100644 index 000000000..d2e494107 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImpl.java @@ -0,0 +1,43 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.StringUtil; + +public class DefaultMatchKeyBuilderImpl implements MatchKeyBuilder { + + @Override + public boolean isSupported(MockCategoryType categoryType) { + return true; + } + + /** + * category + operationName + requestType + */ + @Override + public int getFuzzyMatchKey(Mocker mocker) { + return StringUtil.encodeAndHash( + mocker.getCategoryType().getName(), + mocker.getOperationName(), + mocker.getTargetRequest().getType()); + } + + /** + * operationName + requestBody + * (operationName can remove, but need compatible with old logic, so keep it) + */ + @Override + public int getAccurateMatchKey(Mocker mocker) { + return StringUtil.encodeAndHash( + mocker.getOperationName(), + mocker.getTargetRequest().getBody()); + } + + /** + * requestBody + */ + @Override + public String getEigenBody(Mocker mocker) { + return mocker.getTargetRequest().getBody(); + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImpl.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImpl.java new file mode 100644 index 000000000..5f99a59f0 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImpl.java @@ -0,0 +1,58 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.inst.runtime.model.ArexConstants; +import io.arex.inst.runtime.serializer.Serializer; + +import java.util.HashMap; +import java.util.Map; + +public class HttpClientMatchKeyBuilderImpl implements MatchKeyBuilder { + + @Override + public boolean isSupported(MockCategoryType categoryType) { + return MockCategoryType.HTTP_CLIENT.getName().equals(categoryType.getName()); + } + + /** + * category + operationName + httpMethod + */ + @Override + public int getFuzzyMatchKey(Mocker mocker) { + String operationName = mocker.getOperationName(); + String httpMethod = mocker.getTargetRequest().attributeAsString(ArexConstants.HTTP_METHOD); + return StringUtil.encodeAndHash( + mocker.getCategoryType().getName(), + operationName, + httpMethod); + } + + + /** + * queryString + requestBody + */ + @Override + public int getAccurateMatchKey(Mocker mocker) { + String queryString = mocker.getTargetRequest().attributeAsString(ArexConstants.HTTP_QUERY_STRING); + String requestBody = mocker.getTargetRequest().getBody(); + return StringUtil.encodeAndHash( + queryString, + requestBody); + } + + /** + * queryString + requestBody + */ + @Override + public String getEigenBody(Mocker mocker) { + String queryString = mocker.getTargetRequest().attributeAsString(ArexConstants.HTTP_QUERY_STRING); + Map objectNode = new HashMap<>(); + if (StringUtil.isNotEmpty(queryString)) { + objectNode.put(ArexConstants.HTTP_QUERY_STRING, queryString); + } + objectNode.put(ArexConstants.HTTP_BODY, mocker.getTargetRequest().getBody()); + return Serializer.serialize(objectNode); + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/MatchKeyBuilder.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/MatchKeyBuilder.java new file mode 100644 index 000000000..aefc437bb --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/key/MatchKeyBuilder.java @@ -0,0 +1,14 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; + +public interface MatchKeyBuilder { + boolean isSupported(MockCategoryType categoryType); + + int getFuzzyMatchKey(Mocker mocker); + + int getAccurateMatchKey(Mocker mocker); + + String getEigenBody(Mocker mocker); +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AbstractMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AbstractMatchStrategy.java similarity index 52% rename from arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AbstractMatchStrategy.java rename to arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AbstractMatchStrategy.java index e2fd7daad..9f6de5147 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AbstractMatchStrategy.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AbstractMatchStrategy.java @@ -1,6 +1,8 @@ -package io.arex.inst.runtime.match; +package io.arex.inst.runtime.match.strategy; +import io.arex.agent.bootstrap.model.Mocker; import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.match.MatchStrategyContext; public abstract class AbstractMatchStrategy { static final String MATCH_TITLE = "replay.match.fail"; @@ -22,8 +24,18 @@ private boolean support(MatchStrategyContext context) { return internalCheck(context); } - boolean internalCheck(MatchStrategyContext context) { + public boolean internalCheck(MatchStrategyContext context) { return true; } - abstract void process(MatchStrategyContext context) throws Exception; + + public abstract void process(MatchStrategyContext context) throws Exception; + + public void setContextResult(MatchStrategyContext context, Mocker resultMocker, String failReason) { + if (resultMocker != null) { + resultMocker.setMatched(true); + } else { + context.setReason(failReason); + } + context.setMatchMocker(resultMocker); + } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AccurateMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AccurateMatchStrategy.java similarity index 60% rename from arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AccurateMatchStrategy.java rename to arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AccurateMatchStrategy.java index 2b87fab29..279359e50 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/AccurateMatchStrategy.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/AccurateMatchStrategy.java @@ -1,10 +1,11 @@ -package io.arex.inst.runtime.match; +package io.arex.inst.runtime.match.strategy; import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.inst.runtime.match.MatchKeyFactory; +import io.arex.inst.runtime.match.MatchStrategyContext; import io.arex.inst.runtime.model.MatchStrategyEnum; -import io.arex.inst.runtime.util.MockUtils; import java.util.ArrayList; import java.util.List; @@ -15,50 +16,46 @@ public class AccurateMatchStrategy extends AbstractMatchStrategy{ * priority: * 1. if matching and not matched before return directly * 2. if matched before and find-last mode, return matched one - * 3. if matched multiple result, give next fuzzy match + * 3. if matched multiple result, give next match * 4. if strict match mode and not matched, interrupt */ - void process(MatchStrategyContext context) { + public void process(MatchStrategyContext context) { context.setMatchStrategy(MatchStrategyEnum.ACCURATE); Mocker requestMocker = context.getRequestMocker(); - List replayList = context.getReplayList(); + List recordList = context.getRecordList(); // operationName + requestBody - int methodSignatureHash = MockUtils.methodSignatureHash(requestMocker); - List matchedList = new ArrayList<>(replayList.size()); - for (Mocker mocker : replayList) { - if (methodSignatureHash == mocker.getAccurateMatchKey()) { - matchedList.add(mocker); + requestMocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(requestMocker)); + List matchedList = new ArrayList<>(recordList.size()); + for (Mocker recordMocker : recordList) { + if (requestMocker.getAccurateMatchKey() == recordMocker.getAccurateMatchKey()) { + matchedList.add(recordMocker); } } int matchedCount = matchedList.size(); - if (matchedCount == 1) { Mocker matchMocker = matchedList.get(0); // unmatched or matched but find-last mode (eg: dynamicClass) if (!matchMocker.isMatched() || MockStrategyEnum.FIND_LAST == context.getMockStrategy()) { matchMocker.setMatched(true); context.setMatchMocker(matchMocker); - } else { - context.setReason("accurate match one result, but it has already been matched before, so cannot be used"); + context.setInterrupt(true); + return; } - // other modes can only be matched once, so interrupt and not continue next fuzzy match - context.setInterrupt(true); - return; } - // matched multiple result(like as redis: incr、decr) only retain matched item for next fuzzy match + // matched multiple result(like as redis: incr、decr) only retain matched item for next match if (matchedCount > 1) { - context.setReplayList(matchedList); + context.setRecordList(matchedList); return; } - // if strict match mode and not matched, interrupt and not continue next fuzzy match + // if strict match mode and not matched, interrupt and not continue next match if (MockStrategyEnum.STRICT_MATCH == context.getMockStrategy()) { context.setInterrupt(true); } } @Override - boolean internalCheck(MatchStrategyContext context) { - // if no request params, do next fuzzy match directly + public boolean internalCheck(MatchStrategyContext context) { + // if no request params, do next match directly return StringUtil.isNotEmpty(context.getRequestMocker().getTargetRequest().getBody()); } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/EigenMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/EigenMatchStrategy.java new file mode 100644 index 000000000..db149e717 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/EigenMatchStrategy.java @@ -0,0 +1,112 @@ +package io.arex.inst.runtime.match.strategy; + +import io.arex.agent.bootstrap.model.MockStrategyEnum; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.agent.bootstrap.util.MapUtils; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.agent.compare.model.eigen.EigenOptions; +import io.arex.agent.compare.model.eigen.EigenResult; +import io.arex.agent.compare.sdk.EigenCalculateSDK; +import io.arex.inst.runtime.config.Config; +import io.arex.inst.runtime.match.MatchKeyFactory; +import io.arex.inst.runtime.match.MatchStrategyContext; +import io.arex.inst.runtime.model.CompareConfigurationEntity; +import io.arex.inst.runtime.model.CompareConfigurationEntity.ConfigComparisonExclusionsEntity; +import io.arex.inst.runtime.model.MatchStrategyEnum; + +import java.util.*; + +public class EigenMatchStrategy extends AbstractMatchStrategy{ + + private static final EigenCalculateSDK EIGEN_SDK = new EigenCalculateSDK(); + + /** + * search by eigen value of request + */ + public void process(MatchStrategyContext context) { + context.setMatchStrategy(MatchStrategyEnum.EIGEN); + Mocker replayMocker = context.getRequestMocker(); + List recordList = context.getRecordList(); + Mocker resultMocker = null; + TreeMap> coincidePathMap = new TreeMap<>(); + // calculate all coincide path by eigen value + EigenOptions eigenOptions = getEigenOptions(replayMocker); + calculateEigen(replayMocker, eigenOptions); + for (Mocker recordMocker : recordList) { + if (recordMocker.isMatched()) { + continue; + } + calculateEigen(recordMocker, eigenOptions); + int coincidePath = coincidePath(replayMocker.getEigenMap(), recordMocker.getEigenMap()); + coincidePathMap.computeIfAbsent(coincidePath, k -> new ArrayList<>()).add(recordMocker); + } + + if (MapUtils.isEmpty(coincidePathMap)) { + if (MockStrategyEnum.FIND_LAST == context.getMockStrategy()) { + resultMocker = recordList.get(recordList.size() - 1); + } + } else { + // get the max coincide path (first one in list, default order by creationTime) + resultMocker = coincidePathMap.lastEntry().getValue().get(0); + } + setContextResult(context, resultMocker, "new call, eigen match no result, all has been used"); + } + + private void calculateEigen(Mocker mocker, EigenOptions options) { + if (MapUtils.isNotEmpty(mocker.getEigenMap())) { + return; + } + String eigenBody = MatchKeyFactory.INSTANCE.getEigenBody(mocker); + if (StringUtil.isEmpty(eigenBody)) { + return; + } + + EigenResult eigenResult = EIGEN_SDK.calculateEigen(eigenBody, options); + if (eigenResult == null) { + return; + } + mocker.setEigenMap(eigenResult.getEigenMap()); + } + + private int coincidePath(Map replayEigenMap, Map recordEigenMap) { + int row = 0; + if (MapUtils.isEmpty(replayEigenMap) || MapUtils.isEmpty(recordEigenMap)) { + return row; + } + + for (Map.Entry recordEigenEntry : recordEigenMap.entrySet()) { + Long recordPathValue = recordEigenEntry.getValue(); + Long replayPathValue = replayEigenMap.get(recordEigenEntry.getKey()); + if (Objects.equals(recordPathValue, replayPathValue)) { + row ++; + } + } + return row; + } + + private EigenOptions getEigenOptions(Mocker mocker) { + EigenOptions options = new EigenOptions(); + options.setCategoryType(mocker.getCategoryType().getName()); + CompareConfigurationEntity compareConfiguration = Config.get().getCompareConfiguration(); + if (compareConfiguration == null) { + return options; + } + Set> exclusions = new HashSet<>(); + List comparisonExclusions = compareConfiguration.getComparisonExclusions(); + if (CollectionUtil.isNotEmpty(comparisonExclusions)) { + for (ConfigComparisonExclusionsEntity exclusion : comparisonExclusions) { + if (exclusion == null) { + continue; + } + if (mocker.getCategoryType().getName().equalsIgnoreCase(exclusion.getCategoryType()) + && mocker.getOperationName().equalsIgnoreCase(exclusion.getOperationName())) { + exclusions.addAll(exclusion.getExclusionList()); + } + } + } + options.setIgnoreNodes(compareConfiguration.getIgnoreNodeSet()); + options.setExclusions(CollectionUtil.isNotEmpty(exclusions) ? exclusions : compareConfiguration.getGlobalExclusionList()); + return options; + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/FuzzyMatchStrategy.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/FuzzyMatchStrategy.java new file mode 100644 index 000000000..5df413191 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/match/strategy/FuzzyMatchStrategy.java @@ -0,0 +1,39 @@ +package io.arex.inst.runtime.match.strategy; + +import io.arex.agent.bootstrap.model.MockStrategyEnum; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.inst.runtime.match.MatchStrategyContext; +import io.arex.inst.runtime.model.MatchStrategyEnum; + +import java.util.List; + +public class FuzzyMatchStrategy extends AbstractMatchStrategy { + /** + * search under the same method signature + * replayList is arranged in ascending order by creationTime + * @return not matched before or last one + */ + public void process(MatchStrategyContext context) { + context.setMatchStrategy(MatchStrategyEnum.FUZZY); + List recordList = context.getRecordList(); + Mocker resultMocker = null; + int size = recordList.size(); + for (int i = 0; i < size; i++) { + Mocker recordMocker = recordList.get(i); + if (!recordMocker.isMatched()) { + resultMocker = recordMocker; + break; + } + } + if (resultMocker == null && MockStrategyEnum.FIND_LAST == context.getMockStrategy()) { + resultMocker = recordList.get(size - 1); + } + setContextResult(context, resultMocker, "new call, fuzzy match no result, all has been used"); + } + + @Override + public boolean internalCheck(MatchStrategyContext context) { + return CollectionUtil.isNotEmpty(context.getRecordList()); + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ArexConstants.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ArexConstants.java index 072551d07..0ec86e9fa 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ArexConstants.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ArexConstants.java @@ -68,8 +68,17 @@ private ArexConstants() {} public static final String DATABASE = "database"; public static final String DB_NAME = "dbName"; public static final String DB_PARAMETERS = "parameters"; - public static final String MERGE_MOCKER_TYPE = "java.util.ArrayList-io.arex.agent.bootstrap.model.ArexMocker"; + public static final String DB_SQL = "sql"; public static final int DB_SQL_MAX_LEN = 5000; + public static final String MOCKER_TYPE = "java.util.ArrayList-io.arex.agent.bootstrap.model.ArexMocker"; + public static final String REPLAY_COMPARE_TYPE = "java.util.ArrayList-io.arex.inst.runtime.model.ReplayCompareResultDTO"; + public static final String HTTP_QUERY_STRING = "QueryString"; + public static final String HTTP_METHOD = "HttpMethod"; + public static final String HTTP_BODY = "body"; + public static final String HTTP_CONTENT_TYPE = "ContentType"; public static final String DISABLE_SQL_PARSE = "arex.disable.sql.parse"; + public static final String MATCH_LOG_TITLE = "replay.match"; + public static final String MOCKER_TARGET_TYPE = "io.arex.agent.bootstrap.model.Mocker$Target"; public static final String SPRING_SCAN_PACKAGES = "arex.spring.scan.packages"; + public static final String REPLAY_END_FLAG = "arex-replay-end"; } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/CompareConfigurationEntity.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/CompareConfigurationEntity.java new file mode 100644 index 000000000..cd6adbded --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/CompareConfigurationEntity.java @@ -0,0 +1,67 @@ +package io.arex.inst.runtime.model; + +import java.util.List; +import java.util.Set; + +public class CompareConfigurationEntity { + + private List comparisonExclusions; + + private Set> globalExclusionList; + + private Set ignoreNodeSet; + + public List getComparisonExclusions() { + return comparisonExclusions; + } + + public void setComparisonExclusions(List comparisonExclusions) { + this.comparisonExclusions = comparisonExclusions; + } + + public Set> getGlobalExclusionList() { + return globalExclusionList; + } + + public void setGlobalExclusionList(Set> globalExclusionList) { + this.globalExclusionList = globalExclusionList; + } + + public Set getIgnoreNodeSet() { + return ignoreNodeSet; + } + + public void setIgnoreNodeSet(Set ignoreNodeSet) { + this.ignoreNodeSet = ignoreNodeSet; + } + + public static class ConfigComparisonExclusionsEntity { + private String operationName; + private String categoryType; + private Set> exclusionList; + + public String getOperationName() { + return operationName; + } + + public void setOperationName(String operationName) { + this.operationName = operationName; + } + + public String getCategoryType() { + return categoryType; + } + + public void setCategoryType(String categoryType) { + this.categoryType = categoryType; + } + + public Set> getExclusionList() { + return exclusionList; + } + + public void setExclusionList(Set> exclusionList) { + this.exclusionList = exclusionList; + } + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/QueryAllMockerDTO.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/QueryAllMockerDTO.java index 41c79f792..1bb984b6b 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/QueryAllMockerDTO.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/QueryAllMockerDTO.java @@ -1,14 +1,12 @@ package io.arex.inst.runtime.model; -import io.arex.agent.bootstrap.model.MockCategoryType; - import java.util.Arrays; public class QueryAllMockerDTO { private String recordId; private String replayId; - private String[] fieldNames = new String[]{"id", "categoryType", "operationName", "targetRequest", "targetResponse", "creationTime"}; - private String[] categoryTypes = new String[]{MockCategoryType.DYNAMIC_CLASS.getName(), MockCategoryType.REDIS.getName()}; + private String[] fieldNames =new String[]{"id", "recordId", "categoryType", "operationName", "targetRequest", "targetResponse", "creationTime"}; + private String[] categoryTypes; public String getRecordId() { return recordId; diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ReplayCompareResultDTO.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ReplayCompareResultDTO.java new file mode 100644 index 000000000..faade9403 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/model/ReplayCompareResultDTO.java @@ -0,0 +1,141 @@ +package io.arex.inst.runtime.model; + +import io.arex.agent.bootstrap.model.MockCategoryType; + +public class ReplayCompareResultDTO { + private CategoryTypeDTO categoryType; + private String operationName; + private String recordId; + private String replayId; + private long recordTime; + private long replayTime; + private String recordMessage; + private String replayMessage; + private String appId; + private boolean sameMessage; + private String extendMessage; + + public CategoryTypeDTO getCategoryType() { + return categoryType; + } + + public void setCategoryType(MockCategoryType categoryType) { + this.categoryType = new CategoryTypeDTO(categoryType.getName(), categoryType.isEntryPoint(), categoryType.isSkipComparison()); + } + + public String getOperationName() { + return operationName; + } + + public void setOperationName(String operationName) { + this.operationName = operationName; + } + + public String getRecordId() { + return recordId; + } + + public void setRecordId(String recordId) { + this.recordId = recordId; + } + + public String getReplayId() { + return replayId; + } + + public void setReplayId(String replayId) { + this.replayId = replayId; + } + + public long getRecordTime() { + return recordTime; + } + + public void setRecordTime(long recordTime) { + this.recordTime = recordTime; + } + + public long getReplayTime() { + return replayTime; + } + + public void setReplayTime(long replayTime) { + this.replayTime = replayTime; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getRecordMessage() { + return recordMessage; + } + + public void setRecordMessage(String recordMessage) { + this.recordMessage = recordMessage; + } + + public String getReplayMessage() { + return replayMessage; + } + + public void setReplayMessage(String replayMessage) { + this.replayMessage = replayMessage; + } + + public boolean isSameMessage() { + return sameMessage; + } + + public void setSameMessage(boolean sameMessage) { + this.sameMessage = sameMessage; + } + + public String getExtendMessage() { + return extendMessage; + } + + public void setExtendMessage(String extendMessage) { + this.extendMessage = extendMessage; + } + + static class CategoryTypeDTO { + private String name; + private boolean entryPoint; + private boolean skipComparison; + + CategoryTypeDTO(String name, boolean entryPoint, boolean skipComparison) { + this.name = name; + this.entryPoint = entryPoint; + this.skipComparison = skipComparison; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean isEntryPoint() { + return entryPoint; + } + + public void setEntryPoint(boolean entryPoint) { + this.entryPoint = entryPoint; + } + + public boolean isSkipComparison() { + return skipComparison; + } + + public void setSkipComparison(boolean skipComparison) { + this.skipComparison = skipComparison; + } + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataCollector.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataCollector.java index b36ae14fa..e4d576c5a 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataCollector.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataCollector.java @@ -10,6 +10,8 @@ public interface DataCollector { void save(List mockerList); + void saveReplayCompareResult(String postData); + void invalidCase(String postData); String query(String postData, MockStrategyEnum mockStrategy); diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataService.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataService.java index 02da1a0d7..949fcc099 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataService.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/service/DataService.java @@ -20,6 +20,10 @@ public void save(List mockerList) { saver.save(mockerList); } + public void saveReplayCompareResult(String postData) { + saver.saveReplayCompareResult(postData); + } + public void invalidCase(String postData) { saver.invalidCase(postData); } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/DatabaseUtils.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/DatabaseUtils.java index 2b02f5d5d..1f722efd1 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/DatabaseUtils.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/DatabaseUtils.java @@ -4,8 +4,6 @@ import io.arex.agent.bootstrap.util.StringUtil; import io.arex.agent.thirdparty.util.sqlparser.JSqlParserUtil; import io.arex.agent.thirdparty.util.sqlparser.TableSchema; -import io.arex.inst.runtime.config.Config; -import io.arex.inst.runtime.log.LogManager; import io.arex.inst.runtime.model.ArexConstants; import java.util.*; @@ -14,12 +12,52 @@ public class DatabaseUtils { private static final String DELIMITER = "@"; + public static String parseDbName(String operationName, String dbName) { + if (StringUtil.isNotEmpty(dbName)) { + return dbName; + } + + if (StringUtil.isEmpty(operationName)) { + return operationName; + } + int index = operationName.indexOf('@'); + if (index == -1) { + return operationName; + } + return operationName.substring(0, index); + } + + /** + * @param operationName eg: db1@table1,table2@select@operation1;db2@table3,table4@select@operation2; + * @return tableNames eg: ["table1,table2", "table3,table4"] + */ + public static List parseTableNames(String operationName) { + if (StringUtil.isEmpty(operationName)) { + return Collections.emptyList(); + } + int countMatches = StringUtil.countMatches(operationName, "@"); + if (countMatches < 2) { + return Collections.emptyList(); + } + + String[] operations = StringUtil.split(operationName, ';'); + List tableList = new ArrayList<>(operations.length); + for (String operation : operations) { + String[] subOperation = StringUtil.split(operation, '@', true); + if (subOperation.length < 2 || StringUtil.isEmpty(subOperation[1])) { + continue; + } + tableList.add(subOperation[1]); + } + return tableList; + } + /** * format: dbName@tableNames@action@operationName * eg: db1@table1,table2@select@operation1;db2@table3,table4@select@operation2; */ public static String regenerateOperationName(String dbName, String operationName, String sqlText) { - if (disableSqlParse() || StringUtil.isEmpty(sqlText) || operationName.contains(DELIMITER)) { + if (StringUtil.isEmpty(sqlText) || operationName.contains(DELIMITER)) { return operationName; } @@ -29,22 +67,17 @@ public static String regenerateOperationName(String dbName, String operationName if (StringUtil.isEmpty(sql) || sql.length() > ArexConstants.DB_SQL_MAX_LEN || StringUtil.startWith(sql.toLowerCase(), "exec sp")) { // if exceed the threshold, too large may be due parse stack overflow - LogManager.warn("sql.parse.fail", "invalid sql, too large or sp"); continue; } - try{ - TableSchema tableSchema = JSqlParserUtil.parse(sql); - tableSchema.setDbName(dbName); - operationNames.add(regenerateOperationName(tableSchema, operationName)); - } catch (Throwable ignore) { - // ignore parse failure - } + TableSchema tableSchema = JSqlParserUtil.parse(sql); + tableSchema.setDbName(dbName); + operationNames.add(regenerateOperationName(tableSchema, operationName)); } if (CollectionUtil.isEmpty(operationNames)) { return operationName; } - // ensure that the order of multiple SQL statements is the same - operationNames.sort(String::compareTo); + // sort operation name + Collections.sort(operationNames); return StringUtil.join(operationNames, ";"); } @@ -55,9 +88,5 @@ private static String regenerateOperationName(TableSchema tableSchema, String or .append(originOperationName) .toString(); } - - public static boolean disableSqlParse() { - return Config.get().getBoolean(ArexConstants.DISABLE_SQL_PARSE, Boolean.getBoolean(ArexConstants.DISABLE_SQL_PARSE)); - } } diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MergeRecordUtil.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MergeRecordUtil.java index 2282f354c..a6ee93f9c 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MergeRecordUtil.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MergeRecordUtil.java @@ -14,9 +14,7 @@ import java.util.concurrent.LinkedBlockingQueue; /** - * To solve the problem of insufficient consume capacity of downstream DataServices. - * by controlling the speed at which producers produce mockers. - * multiple producer -> single consumer + * merge record and replay util */ public class MergeRecordUtil { private static final AgentSizeOf agentSizeOf = AgentSizeOf.newInstance(); @@ -26,7 +24,7 @@ private MergeRecordUtil() {} public static void mergeRecord(Mocker requestMocker) { List> mergeList = merge(requestMocker); for (List mergeRecords : mergeList) { - MockUtils.executeRecord(mergeRecords); + MockManager.executeRecord(mergeRecords); } } @@ -130,7 +128,7 @@ public static void recordRemain(ArexContext context) { mergeRecordQueue.drainTo(mergeRecordList); List> splitList = checkAndSplit(mergeRecordList); for (List mergeRecords : splitList) { - MockUtils.executeRecord(mergeRecords); + MockManager.executeRecord(mergeRecords); } } catch (Exception e) { LogManager.warn("merge.record.remain.error", e); diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockManager.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockManager.java new file mode 100644 index 000000000..2cebf3f69 --- /dev/null +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockManager.java @@ -0,0 +1,27 @@ +package io.arex.inst.runtime.util; + +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.context.ArexContext; + +import java.util.List; + +/** + * decouple, resolve cycle dependency + */ +public class MockManager { + public static void mergeRecord(Mocker requestMocker) { + MergeRecordUtil.mergeRecord(requestMocker); + } + + public static void recordRemain(ArexContext context) { + MergeRecordUtil.recordRemain(context); + } + + public static void executeRecord(List mockerList) { + MockUtils.executeRecord(mockerList); + } + + public static void saveReplayRemainCompareRelation(ArexContext context) { + ReplayUtil.saveRemainCompareResult(context); + } +} diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockUtils.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockUtils.java index 90da2fe64..b2a81999e 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockUtils.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/MockUtils.java @@ -14,6 +14,7 @@ import io.arex.inst.runtime.match.ReplayMatcher; import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.QueryAllMockerDTO; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; import io.arex.inst.runtime.service.DataService; import io.arex.inst.runtime.util.sizeof.AgentSizeOf; @@ -99,7 +100,7 @@ public static void recordMocker(Mocker requestMocker) { return; } if (requestMocker.isNeedMerge() && enableMergeRecord()) { - MergeRecordUtil.mergeRecord(requestMocker); + MockManager.mergeRecord(requestMocker); return; } @@ -107,7 +108,7 @@ public static void recordMocker(Mocker requestMocker) { if (requestMocker.getCategoryType().isEntryPoint()) { // after main entry record finished, record remain merge mocker that have not reached the merge threshold once(such as dynamicClass) - MergeRecordUtil.recordRemain(ContextManager.currentContext()); + MockManager.recordRemain(ContextManager.currentContext()); } } @@ -131,14 +132,11 @@ public static Mocker replayMocker(Mocker requestMocker, MockStrategyEnum mockStr return null; } - if (requestMocker.isNeedMerge()) { - Mocker matchMocker = ReplayMatcher.match(requestMocker, mockStrategy); - // compatible with old version(fixed case without merge) - if (matchMocker != null) { - return matchMocker; - } + if (isNotConfigFile(requestMocker.getCategoryType())) { + return ReplayMatcher.match(requestMocker, mockStrategy); } + // direct replay not depends on cache(ContextManager.currentContext().cachedReplayResultMap), eg:config file return executeReplay(requestMocker, mockStrategy); } @@ -216,19 +214,6 @@ public static boolean checkResponseMocker(Mocker responseMocker) { return true; } - public static int methodSignatureHash(Mocker requestMocker) { - return StringUtil.encodeAndHash(String.format("%s_%s", - requestMocker.getOperationName(), - requestMocker.getTargetRequest().getBody())); - } - - public static int methodRequestTypeHash(Mocker requestMocker) { - return StringUtil.encodeAndHash(String.format("%s_%s_%s", - requestMocker.getCategoryType().getName(), - requestMocker.getOperationName(), - requestMocker.getTargetRequest().getType())); - } - /** * get all mockers under current one case */ @@ -238,8 +223,7 @@ public static List queryMockers(QueryAllMockerDTO requestMocker) { String data = DataService.INSTANCE.queryAll(postJson); String cost = String.valueOf(System.currentTimeMillis() - startTime); if (StringUtil.isEmpty(data) || EMPTY_JSON.equals(data)) { - LogManager.warn(requestMocker.replayLogTitle(), - StringUtil.format("cost: %s ms%nrequest: %s%nresponse is null.", cost, postJson)); + LogManager.warn(requestMocker.replayLogTitle(), StringUtil.format("response is null. cost: %s ms, request: %s", cost, postJson)); return null; } String message = StringUtil.format("cost: %s ms%nrequest: %s", cost, postJson); @@ -248,7 +232,17 @@ public static List queryMockers(QueryAllMockerDTO requestMocker) { } LogManager.info(requestMocker.replayLogTitle(), message); - return Serializer.deserialize(data, ArexConstants.MERGE_MOCKER_TYPE); + return Serializer.deserialize(data, ArexConstants.MOCKER_TYPE); + } + + public static void saveReplayCompareResult(ArexContext context, List replayCompareList) { + String postData = Serializer.serialize(replayCompareList); + String message = StringUtil.format("compare count: %s", String.valueOf(replayCompareList.size())); + if (Config.get().isEnableDebug()) { + message = StringUtil.format(message + "%nrequest: %s", postData); + } + LogManager.info(context, "saveReplayCompareResult", message); + DataService.INSTANCE.saveReplayCompareResult(postData); } private static boolean enableMergeRecord() { diff --git a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/ReplayUtil.java b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/ReplayUtil.java index be24e45a6..3437c5c6b 100644 --- a/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/ReplayUtil.java +++ b/arex-instrumentation-api/src/main/java/io/arex/inst/runtime/util/ReplayUtil.java @@ -4,33 +4,31 @@ import io.arex.agent.bootstrap.model.MockCategoryType; import io.arex.agent.bootstrap.model.Mocker; import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.agent.thirdparty.util.CompressUtil; +import io.arex.inst.runtime.config.Config; +import io.arex.inst.runtime.context.ArexContext; import io.arex.inst.runtime.context.ContextManager; import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.match.MatchKeyFactory; import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.MergeDTO; import io.arex.inst.runtime.model.QueryAllMockerDTO; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; - import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Predicate; /** - * Replay all mockers under case - * (called once after arexContext init) + * ReplayUtil */ public class ReplayUtil { - - private static final Predicate FILTER_MERGE_RECORD = mocker -> - ArexConstants.MERGE_RECORD_NAME.equals(mocker.getOperationName()); - - private static final Predicate FILTER_MERGE_TYPE = mergeDTO -> - !MockCategoryType.DYNAMIC_CLASS.getName().equals(mergeDTO.getCategory()) - && !MockCategoryType.REDIS.getName().equals(mergeDTO.getCategory()); - /** - * init replay all mocker under case and cached replay result + * init replay all mockers under case and cached replay result at context */ public static void queryMockers() { if (!ContextManager.needReplay()) { @@ -40,33 +38,35 @@ public static void queryMockers() { QueryAllMockerDTO requestMocker = new QueryAllMockerDTO(); requestMocker.setRecordId(ContextManager.currentContext().getCaseId()); requestMocker.setReplayId(ContextManager.currentContext().getReplayId()); - List allMockerList = MockUtils.queryMockers(requestMocker); - if (CollectionUtil.isEmpty(allMockerList)) { + List recordMockerList = MockUtils.queryMockers(requestMocker); + if (CollectionUtil.isEmpty(recordMockerList)) { return; } - filterMergeMocker(allMockerList); - - Map> cachedReplayResultMap = ContextManager.currentContext().getCachedReplayResultMap(); - cachedReplayResultMap.clear(); - buildReplayResultMap(allMockerList, cachedReplayResultMap); - - ascendingSortByCreationTime(cachedReplayResultMap); + filterMocker(recordMockerList); + Map> cachedReplayMap = ContextManager.currentContext().getCachedReplayResultMap(); + cachedReplayMap.clear(); + buildReplayResultMap(recordMockerList, cachedReplayMap); + ascendingSortByCreationTime(cachedReplayMap); } catch (Exception e) { - LogManager.warn("replay.all.mocker", e); + LogManager.warn("replay.allMocker", e); } } /** - * compatible with merge record, after batchSave published can be removed + * compatible with merge record, after batchSave publish can be removed */ - private static void filterMergeMocker(List allMockerList) { + private static void filterMocker(List allMockerList) { List splitMockerList = new ArrayList<>(); - for (Mocker mergeMocker : allMockerList) { - if (!FILTER_MERGE_RECORD.test(mergeMocker)) { + Predicate filterMergeRecord = mocker -> ArexConstants.MERGE_RECORD_NAME.equals(mocker.getOperationName()); + for (Mocker mocker : allMockerList) { + // decompress zstd data + decompress(mocker); + + if (!filterMergeRecord.test(mocker)) { continue; } - List mergeReplayList = Serializer.deserialize(mergeMocker.getTargetResponse().getBody(), ArexConstants.MERGE_TYPE); + List mergeReplayList = Serializer.deserialize(mocker.getTargetResponse().getBody(), ArexConstants.MERGE_TYPE); if (CollectionUtil.isEmpty(mergeReplayList)) { continue; } @@ -75,14 +75,29 @@ private static void filterMergeMocker(List allMockerList) { if (CollectionUtil.isEmpty(splitMockerList)) { return; } - allMockerList.removeIf(FILTER_MERGE_RECORD); + allMockerList.removeIf(filterMergeRecord); allMockerList.addAll(splitMockerList); } + private static void decompress(Mocker mocker) { + // decompress zstd data + String originalRequest = mocker.getRequest(); + if (StringUtil.isNotEmpty(originalRequest)) { + originalRequest = CompressUtil.zstdDecompress(Base64.getDecoder().decode(originalRequest)); + mocker.setTargetRequest(Serializer.deserialize(originalRequest, ArexConstants.MOCKER_TARGET_TYPE)); + } + String originalResponse = mocker.getResponse(); + if (StringUtil.isNotEmpty(originalResponse)) { + originalResponse = CompressUtil.zstdDecompress(Base64.getDecoder().decode(originalResponse)); + mocker.setTargetResponse(Serializer.deserialize(originalResponse, ArexConstants.MOCKER_TARGET_TYPE)); + } + } + private static List convertMergeMocker(List mergeReplayList) { List convertMockerList = new ArrayList<>(); for (MergeDTO mergeDTO : mergeReplayList) { - if (mergeDTO == null || FILTER_MERGE_TYPE.test(mergeDTO)) { + if (mergeDTO == null || (!MockCategoryType.DYNAMIC_CLASS.getName().equals(mergeDTO.getCategory()) + && !MockCategoryType.REDIS.getName().equals(mergeDTO.getCategory()))) { continue; } ArexMocker mocker = MockUtils.create(MockCategoryType.of(mergeDTO.getCategory()), mergeDTO.getOperationName()); @@ -99,19 +114,78 @@ private static List convertMergeMocker(List mergeReplayList) { return convertMockerList; } - private static void buildReplayResultMap(List replayMockers, Map> cachedReplayResultMap) { - for (Mocker replayMocker : replayMockers) { - if (replayMocker == null) { + /** + * 

+     * format:
+     * {
+     *   fuzzyMatchKeyHash : [Mockers]
+     * }
+     *
+     * demo:
+     * {
+     *   1233213331 : [
+     *                  {
+     *                  "accurateMatchKey": 3454562343,
+     *                  "categoryType": "Httpclient",
+     *                  "operationName": "/order/query",
+     *                  "targetRequest": "...",
+     *                  "targetResponse": "...",
+     *                  ...
+     *                  }
+     *               ]
+     *   4545626535 : [
+     *                  {
+     *                  "accurateMatchKey": 6534247741,
+     *                  "categoryType": "Database",
+     *                  "operationName": "query",
+     *                  "targetRequest": "...",
+     *                  "targetResponse": "...",
+     *                  ...
+     *                  },
+     *                  {
+     *                  "accurateMatchKey": 9866734220,
+     *                  "categoryType": "Database",
+     *                  "operationName": "update",
+     *                  "targetRequest": "...",
+     *                  "targetResponse": "...",
+     *                  ...
+     *                  }
+     *               ]
+     *   ...
+     * }
+     *
+ */ + private static void buildReplayResultMap(List recordMockerList, Map> cachedReplayResultMap) { + for (Mocker recordMocker : recordMockerList) { + if (recordMocker == null) { continue; } + // compatible with fixed case, set operationName for database mocker + compatibleFixedCase(recordMocker); + // replay match need methodRequestTypeHash and methodSignatureHash - if (replayMocker.getFuzzyMatchKey() == 0) { - replayMocker.setFuzzyMatchKey(MockUtils.methodRequestTypeHash(replayMocker)); + if (recordMocker.getFuzzyMatchKey() == 0) { + recordMocker.setFuzzyMatchKey(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(recordMocker)); } - if (replayMocker.getAccurateMatchKey() == 0) { - replayMocker.setAccurateMatchKey(MockUtils.methodSignatureHash(replayMocker)); + // fuzzyMatchKey and accurateMatchKey maybe not 0 on old merge record(methodRequestTypeHash、methodSignatureHash) + if (recordMocker.getAccurateMatchKey() == 0) { + recordMocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(recordMocker)); } - cachedReplayResultMap.computeIfAbsent(replayMocker.getFuzzyMatchKey(), k -> new ArrayList<>()).add(replayMocker); + + // eigen will be calculated in agent + recordMocker.setEigenMap(null); + cachedReplayResultMap.computeIfAbsent(recordMocker.getFuzzyMatchKey(), k -> new ArrayList<>()).add(recordMocker); + } + } + + private static void compatibleFixedCase(Mocker mocker) { + String categoryType = mocker.getCategoryType().getName(); + if (MockCategoryType.DATABASE.getName().equals(categoryType)) { + String dbName = mocker.getTargetRequest().attributeAsString(ArexConstants.DB_NAME); + String sql = mocker.getTargetRequest().getBody(); + // if operationName contains '@' then not need to regenerate + String operationName = DatabaseUtils.regenerateOperationName(StringUtil.defaultString(dbName), mocker.getOperationName(), sql); + mocker.setOperationName(operationName); } } @@ -128,4 +202,99 @@ private static void ascendingSortByCreationTime(Map> cache }); } } + + public static void saveReplayCompareResult() { + ArexContext context = ContextManager.currentContext(); + if (context == null || !context.isReplay()) { + return; + } + LinkedBlockingQueue replayCompareResultQueue = context.getReplayCompareResultQueue(); + if (replayCompareResultQueue.isEmpty()) { + return; + } + List replayCompareList = new ArrayList<>(); + replayCompareResultQueue.drainTo(replayCompareList); + MockUtils.saveReplayCompareResult(context, replayCompareList); + + Map> cachedReplayResultMap = context.getCachedReplayResultMap(); + StringBuilder message = new StringBuilder(); + for (List cachedReplayList : cachedReplayResultMap.values()) { + for (Mocker cachedMocker : cachedReplayList) { + message.append(StringUtil.format("matched: %s, detail: %s %n", + String.valueOf(cachedMocker.isMatched()), cachedMocker.logBuilder().toString())); + } + } + LogManager.info("saveReplayCompareResult", message.toString()); + } + + /** + * Record the compare relationship again when delaying the context cleaning to ensure + * that all replay matches (including asynchronous ones) match correctly. + * And this time we counted call missing, because if call missing is counted on the main interface, + * there may be asynchronous interfaces that have not been matched yet, + * Although it is currently in an unmatched state, it may become a matching state after asynchronous matching, + * so it needs to be counted call missing last. + */ + public static void saveRemainCompareResult(ArexContext context) { + if (context == null) { + return; + } + LinkedBlockingQueue replayCompareResultQueue = context.getReplayCompareResultQueue(); + // find unmatched mockers (call missing) + Map> cachedReplayResultMap = context.getCachedReplayResultMap(); + for (List cachedReplayList : cachedReplayResultMap.values()) { + for (Mocker cachedMocker : cachedReplayList) { + if (cachedMocker.isMatched() || cachedMocker.getCategoryType().isSkipComparison()) { + continue; + } + cachedMocker.setAppId(System.getProperty("arex.service.name")); + cachedMocker.setReplayId(context.getReplayId()); + String recordMsg = getCompareMessage(cachedMocker); + ReplayCompareResultDTO callMissingDTO = convertCompareResult(cachedMocker, recordMsg, + null, cachedMocker.getCreationTime(), Long.MAX_VALUE, false); + boolean success = replayCompareResultQueue.offer(callMissingDTO); + // log call missing + String message = StringUtil.format("%s %n%s", + "match fail, reason: call missing", cachedMocker.logBuilder().toString()); + if (success && Config.get().isEnableDebug()) { + message += StringUtil.format("%ncall missing mocker: %s", Serializer.serialize(cachedMocker)); + } + LogManager.info(context, ArexConstants.MATCH_LOG_TITLE, message); + } + } + if (replayCompareResultQueue.isEmpty()) { + return; + } + List replayCompareList = new ArrayList<>(); + replayCompareResultQueue.drainTo(replayCompareList); + MockUtils.saveReplayCompareResult(context, replayCompareList); + LogManager.info("saveRemainCompareResult", "remain size: " + replayCompareList.size()); + } + + public static ReplayCompareResultDTO convertCompareResult(Mocker replayMocker, String recordMsg, String replayMsg, + long recordTime, long replayTime, boolean sameMsg) { + ReplayCompareResultDTO compareResult = new ReplayCompareResultDTO(); + compareResult.setAppId(replayMocker.getAppId()); + compareResult.setCategoryType(replayMocker.getCategoryType()); + compareResult.setOperationName(replayMocker.getOperationName()); + compareResult.setRecordId(replayMocker.getRecordId()); + compareResult.setReplayId(replayMocker.getReplayId()); + compareResult.setRecordMessage(recordMsg); + compareResult.setReplayMessage(replayMsg); + compareResult.setRecordTime(recordTime); + compareResult.setReplayTime(replayTime); + compareResult.setSameMessage(sameMsg); + return compareResult; + } + + public static String getCompareMessage(Mocker mocker) { + String compareMessage = mocker.getTargetRequest().getBody(); + if (mocker.getCategoryType().isEntryPoint()) { + compareMessage = mocker.getTargetResponse().getBody(); + } else if (MockCategoryType.DATABASE.getName().equals(mocker.getCategoryType().getName()) + || MockCategoryType.MESSAGE_PRODUCER.getName().equals(mocker.getCategoryType().getName())) { + compareMessage = Serializer.serialize(mocker.getTargetRequest()); + } + return compareMessage; + } } diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/listener/EventProcessorTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/listener/EventProcessorTest.java index 840aa70c8..2cd86d341 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/listener/EventProcessorTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/listener/EventProcessorTest.java @@ -1,7 +1,5 @@ package io.arex.inst.runtime.listener; -import static org.mockito.ArgumentMatchers.any; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,10 +12,10 @@ import io.arex.inst.runtime.context.ContextManager; import io.arex.inst.runtime.log.LogManager; import io.arex.inst.runtime.log.Logger; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.serializer.StringSerializable; import io.arex.agent.bootstrap.util.ServiceLoader; -import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Type; @@ -112,6 +110,9 @@ void testInitClass() throws Exception { @Test void onExit() { + ArexContext context = Mockito.mock(ArexContext.class); + Mockito.when(ContextManager.currentContext()).thenReturn(context); + Mockito.when(context.getAttachment(ArexConstants.REPLAY_END_FLAG)).thenReturn(true); Assertions.assertDoesNotThrow(EventProcessor::onExit); } diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AbstractMatchStrategyTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AbstractMatchStrategyTest.java index ffaec44c4..24d2f0b51 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AbstractMatchStrategyTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AbstractMatchStrategyTest.java @@ -1,7 +1,10 @@ package io.arex.inst.runtime.match; import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.match.strategy.AbstractMatchStrategy; +import io.arex.inst.runtime.match.strategy.AccurateMatchStrategy; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -17,7 +20,7 @@ class AbstractMatchStrategyTest { @BeforeAll static void setUp() { target = new AccurateMatchStrategy(); - mocker = new ArexMocker(); + mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); mocker.setTargetResponse(new Mocker.Target()); mocker.setTargetRequest(new Mocker.Target()); mocker.getTargetRequest().setBody("mock"); @@ -31,7 +34,14 @@ static void tearDown() { @Test void match() { assertDoesNotThrow(() -> target.match(null)); - MatchStrategyContext context = new MatchStrategyContext(mocker, new ArrayList<>(), null); + MatchStrategyContext context = new MatchStrategyContext(mocker, null); + context.setRecordList(new ArrayList<>()); assertDoesNotThrow(() -> target.match(context)); } + + @Test + void setContextResult() { + assertDoesNotThrow(() -> target.setContextResult(new MatchStrategyContext(mocker, null), mocker, null)); + assertDoesNotThrow(() -> target.setContextResult(new MatchStrategyContext(mocker, null), null, null)); + } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AccurateMatchStrategyTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AccurateMatchStrategyTest.java index fb6185a7e..1523b40d7 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AccurateMatchStrategyTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/AccurateMatchStrategyTest.java @@ -4,6 +4,7 @@ import io.arex.agent.bootstrap.model.MockCategoryType; import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.match.strategy.AccurateMatchStrategy; import io.arex.inst.runtime.util.MockUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -47,31 +48,46 @@ void process(MatchStrategyContext context, Predicate asser static Stream processCase() { Supplier contextSupplier1 = () -> { - ArexMocker mocker = new ArexMocker(); - mocker.setTargetResponse(new Mocker.Target()); + ArexMocker mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); mocker.setTargetRequest(new Mocker.Target()); - mocker.setCategoryType(MockCategoryType.DYNAMIC_CLASS); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + mocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(mocker)); List mergeReplayList = new ArrayList<>(); - mergeReplayList.add(new ArexMocker()); - return new MatchStrategyContext(mocker, mergeReplayList, MockStrategyEnum.FIND_LAST); + mergeReplayList.add(mocker); + MatchStrategyContext context = new MatchStrategyContext(mocker, MockStrategyEnum.FIND_LAST); + context.setRecordList(mergeReplayList); + return context; }; Supplier contextSupplier2 = () -> { MatchStrategyContext context = contextSupplier1.get(); - context.getReplayList().get(0).setMatched(true); + context.getRecordList().get(0).setMatched(true); context.setMockStrategy(MockStrategyEnum.STRICT_MATCH); return context; }; Supplier contextSupplier3 = () -> { MatchStrategyContext context = contextSupplier1.get(); - context.getReplayList().add(new ArexMocker()); + context.getRecordList().add(new ArexMocker()); return context; }; Supplier contextSupplier4 = () -> { MatchStrategyContext context = contextSupplier1.get(); - context.getReplayList().get(0).setAccurateMatchKey(1); + context.getRecordList().get(0).setFuzzyMatchKey(1); context.setMockStrategy(MockStrategyEnum.STRICT_MATCH); return context; }; + Supplier multiMatchResult = () -> { + MatchStrategyContext context = contextSupplier1.get(); + ArexMocker mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + mocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(mocker)); + context.getRecordList().add(mocker); + return context; + }; Predicate asserts1 = context -> !context.isInterrupt(); Predicate asserts2 = MatchStrategyContext::isInterrupt; @@ -80,7 +96,8 @@ static Stream processCase() { arguments(contextSupplier1.get(), asserts1), arguments(contextSupplier2.get(), asserts2), arguments(contextSupplier3.get(), asserts1), - arguments(contextSupplier4.get(), asserts2) + arguments(contextSupplier4.get(), asserts2), + arguments(multiMatchResult.get(), asserts2) ); } @@ -88,6 +105,6 @@ static Stream processCase() { void internalCheck() { ArexMocker mocker = new ArexMocker(); mocker.setTargetRequest(new Mocker.Target()); - assertFalse(accurateMatchStrategy.internalCheck(new MatchStrategyContext(mocker, null, null))); + assertFalse(accurateMatchStrategy.internalCheck(new MatchStrategyContext(mocker, null))); } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/EigenMatchStrategyTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/EigenMatchStrategyTest.java index bf993f056..10d8fd373 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/EigenMatchStrategyTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/EigenMatchStrategyTest.java @@ -1,21 +1,94 @@ package io.arex.inst.runtime.match; +import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.MockStrategyEnum; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.config.Config; +import io.arex.inst.runtime.match.strategy.EigenMatchStrategy; +import io.arex.inst.runtime.model.CompareConfigurationEntity; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.junit.jupiter.params.provider.Arguments.arguments; class EigenMatchStrategyTest { static EigenMatchStrategy eigenMatchStrategy; + static Config config; @BeforeAll static void setUp() { eigenMatchStrategy = new EigenMatchStrategy(); + Mockito.mockStatic(Config.class); + config = Mockito.mock(Config.class); + Mockito.when(Config.get()).thenReturn(config); } @AfterAll static void tearDown() { eigenMatchStrategy = null; + config = null; + Mockito.clearAllCaches(); + } + + @ParameterizedTest + @MethodSource("processCase") + void process(Runnable mocker, MatchStrategyContext context, Predicate asserts) { + mocker.run(); + eigenMatchStrategy.process(context); + asserts.test(context); + } + + static Stream processCase() { + Runnable emptyMocker = () -> {}; + + Runnable mockerCompareConfig = () -> { + CompareConfigurationEntity compareConfig = new CompareConfigurationEntity(); + CompareConfigurationEntity.ConfigComparisonExclusionsEntity exclusion = new CompareConfigurationEntity.ConfigComparisonExclusionsEntity(); + exclusion.setCategoryType(MockCategoryType.DYNAMIC_CLASS.getName()); + exclusion.setOperationName("mock"); + exclusion.setExclusionList(new HashSet<>(new ArrayList<>())); + compareConfig.setComparisonExclusions(Collections.singletonList(exclusion)); + Mockito.when(config.getCompareConfiguration()).thenReturn(compareConfig); + }; + + Supplier contextSupplier1 = () -> { + ArexMocker mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + mocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(mocker)); + List mergeReplayList = new ArrayList<>(); + mergeReplayList.add(mocker); + MatchStrategyContext context = new MatchStrategyContext(mocker, MockStrategyEnum.FIND_LAST); + context.setRecordList(mergeReplayList); + return context; + }; + Supplier contextSupplier2 = () -> { + MatchStrategyContext context = contextSupplier1.get(); + context.getRecordList().get(0).setMatched(true); + return context; + }; + + Predicate asserts1 = context -> !context.isInterrupt(); + Predicate asserts2 = MatchStrategyContext::isInterrupt; + + return Stream.of( + arguments(emptyMocker, contextSupplier1.get(), asserts1), + arguments(mockerCompareConfig, contextSupplier2.get(), asserts2) + ); } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/FuzzyMatchStrategyTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/FuzzyMatchStrategyTest.java index 375594edb..278802c11 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/FuzzyMatchStrategyTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/FuzzyMatchStrategyTest.java @@ -5,6 +5,7 @@ import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; import io.arex.inst.runtime.config.Config; +import io.arex.inst.runtime.match.strategy.FuzzyMatchStrategy; import io.arex.inst.runtime.util.MockUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -35,29 +36,27 @@ static void tearDown() { @Test void process() { - ArexMocker mocker = new ArexMocker(); - mocker.setTargetResponse(new Mocker.Target()); + ArexMocker mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); mocker.setTargetRequest(new Mocker.Target()); - mocker.setCategoryType(MockCategoryType.DYNAMIC_CLASS); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + mocker.setAccurateMatchKey(MatchKeyFactory.INSTANCE.getAccurateMatchKey(mocker)); List mergeReplayList = new ArrayList<>(); - Mocker mergeDTO = new ArexMocker(); - mergeReplayList.add(mergeDTO); - MatchStrategyContext context =new MatchStrategyContext(mocker, mergeReplayList, MockStrategyEnum.FIND_LAST); + mergeReplayList.add(mocker); + MatchStrategyContext context = new MatchStrategyContext(mocker, MockStrategyEnum.FIND_LAST); + context.setRecordList(mergeReplayList); Mockito.when(Config.get().isEnableDebug()).thenReturn(true); fuzzyMatchStrategy.process(context); assertNotNull(context.getMatchMocker()); - mergeDTO.setMatched(true); + mocker.setMatched(true); fuzzyMatchStrategy.process(context); assertNotNull(context.getMatchMocker()); - - // fuzzy match no result - MatchStrategyContext context1 =new MatchStrategyContext(mocker, mergeReplayList, MockStrategyEnum.OVER_BREAK); - fuzzyMatchStrategy.process(context1); } @Test void internalCheck() { - assertFalse(fuzzyMatchStrategy.internalCheck(new MatchStrategyContext(null, null, null))); + assertFalse(fuzzyMatchStrategy.internalCheck(new MatchStrategyContext(null, null))); } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchKeyFactoryTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchKeyFactoryTest.java new file mode 100644 index 000000000..5cdc2557d --- /dev/null +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchKeyFactoryTest.java @@ -0,0 +1,44 @@ +package io.arex.inst.runtime.match; + +import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class MatchKeyFactoryTest { + + static ArexMocker mocker; + + @BeforeAll + static void setUp() { + mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + } + + @AfterAll + static void tearDown() { + mocker = null; + } + + @Test + void getFuzzyMatchKey() { + assertNotEquals(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(mocker), 0); + } + + @Test + void getAccurateMatchKey() { + assertNotEquals(MatchKeyFactory.INSTANCE.getAccurateMatchKey(mocker), 0); + } + + @Test + void getEigenBody() { + assertNotNull(MatchKeyFactory.INSTANCE.getEigenBody(mocker)); + } +} \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyContextTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyContextTest.java index ed2f347d9..4c8983d7c 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyContextTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyContextTest.java @@ -10,7 +10,7 @@ class MatchStrategyContextTest { @BeforeAll static void setUp() { - context = new MatchStrategyContext(null, null, null); + context = new MatchStrategyContext(null, null); } @Test @@ -24,13 +24,13 @@ void setRequestMocker() { } @Test - void getReplayList() { - assertNull(context.getReplayList()); + void getRecordList() { + assertNull(context.getRecordList()); } @Test - void setReplayList() { - assertDoesNotThrow(() -> context.setReplayList(null)); + void setRecordList() { + assertDoesNotThrow(() -> context.setRecordList(null)); } @Test diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyRegisterTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyRegisterTest.java index 8357b34ea..01be4726a 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyRegisterTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/MatchStrategyRegisterTest.java @@ -1,5 +1,6 @@ package io.arex.inst.runtime.match; +import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockCategoryType; import org.junit.jupiter.api.Test; @@ -9,6 +10,8 @@ class MatchStrategyRegisterTest { @Test void getMatchStrategies() { - assertNotNull(MatchStrategyRegister.getMatchStrategies(MockCategoryType.DYNAMIC_CLASS)); + assertNotNull(MatchStrategyRegister.getMatchStrategies(new ArexMocker(MockCategoryType.DYNAMIC_CLASS), 1)); + assertNotNull(MatchStrategyRegister.getMatchStrategies(new ArexMocker(MockCategoryType.DYNAMIC_CLASS), 2)); + assertNotNull(MatchStrategyRegister.getMatchStrategies(new ArexMocker(MockCategoryType.DATABASE), 2)); } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/ReplayMatcherTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/ReplayMatcherTest.java index 4b9ab32f0..ab6a9f37d 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/ReplayMatcherTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/ReplayMatcherTest.java @@ -2,22 +2,37 @@ import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.MockResult; import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; import io.arex.inst.runtime.config.Config; import io.arex.inst.runtime.context.ArexContext; import io.arex.inst.runtime.context.ContextManager; +import io.arex.inst.runtime.match.strategy.AccurateMatchStrategy; +import io.arex.inst.runtime.util.IgnoreUtils; import io.arex.inst.runtime.util.MockUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockedConstruction; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import java.util.*; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Predicate; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +@ExtendWith(MockitoExtension.class) class ReplayMatcherTest { @BeforeAll @@ -34,27 +49,52 @@ static void tearDown() { Mockito.clearAllCaches(); } - @Test - void match() { + @ParameterizedTest + @MethodSource("matchCase") + void match(Runnable mocker, Mocker requestMocker, Predicate predicate) { + mocker.run(); + Mocker result = ReplayMatcher.match(requestMocker, MockStrategyEnum.FIND_LAST); + assertTrue(predicate.test(result)); + } + + static Stream matchCase() { ArexMocker requestMocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); requestMocker.setOperationName("mock"); requestMocker.setTargetRequest(new Mocker.Target()); + requestMocker.getTargetRequest().setBody("mock"); requestMocker.setTargetResponse(new Mocker.Target()); + ArexContext context = Mockito.mock(ArexContext.class); - Mockito.when(ContextManager.currentContext()).thenReturn(context); Map> cachedReplayResultMap = new HashMap<>(); - Mockito.when(context.getCachedReplayResultMap()).thenReturn(cachedReplayResultMap); - assertNull(ReplayMatcher.match(requestMocker, MockStrategyEnum.FIND_LAST)); - - Mockito.when(MockUtils.methodRequestTypeHash(requestMocker)).thenReturn(1); List mergeReplayList = new ArrayList<>(); - mergeReplayList.add(new ArexMocker()); - cachedReplayResultMap.put(1, mergeReplayList); - Mockito.when(MatchStrategyRegister.getMatchStrategies(any())).thenReturn(Collections.singletonList(new AccurateMatchStrategy())); - Mockito.when(Config.get().isEnableDebug()).thenReturn(true); - assertNull(ReplayMatcher.match(requestMocker, MockStrategyEnum.FIND_LAST)); - // match no result, not exist this method signature - Mockito.when(MockUtils.methodRequestTypeHash(requestMocker)).thenReturn(2); - assertNull(ReplayMatcher.match(requestMocker, MockStrategyEnum.FIND_LAST)); + mergeReplayList.add(requestMocker); + Runnable mockerContext = () -> { + Mockito.when(ContextManager.currentContext()).thenReturn(context); + Mockito.when(context.getCachedReplayResultMap()).thenReturn(cachedReplayResultMap); + }; + Runnable mockerStrategy = () -> { + Mockito.when(MatchStrategyRegister.getMatchStrategies(any(), anyInt())) + .thenReturn(Collections.singletonList(new AccurateMatchStrategy())); + Mockito.when(Config.get().isEnableDebug()).thenReturn(true); + cachedReplayResultMap.put(1, mergeReplayList); + }; + Runnable dynamicCLassRecordListMocker = () -> { + cachedReplayResultMap.put(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(requestMocker), mergeReplayList); + }; + + Runnable dataBaseRecordListMocker = () -> { + requestMocker.setCategoryType(MockCategoryType.DATABASE); + Mockito.when(context.getReplayCompareResultQueue()).thenReturn(new LinkedBlockingQueue<>()); + cachedReplayResultMap.put(MatchKeyFactory.INSTANCE.getFuzzyMatchKey(requestMocker), mergeReplayList); + }; + + Predicate predicate1 = Objects::isNull; + Predicate predicate2 = Objects::nonNull; + return Stream.of( + arguments(mockerContext, requestMocker, predicate1), + arguments(mockerStrategy, requestMocker, predicate1), + arguments(dynamicCLassRecordListMocker, requestMocker, predicate2), + arguments(dataBaseRecordListMocker, requestMocker, predicate2) + ); } } diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImplTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImplTest.java new file mode 100644 index 000000000..66ed94d86 --- /dev/null +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DatabaseMatchKeyBuilderImplTest.java @@ -0,0 +1,66 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.model.ArexConstants; +import io.arex.inst.runtime.serializer.Serializer; +import io.arex.inst.runtime.util.DatabaseUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; + +class DatabaseMatchKeyBuilderImplTest { + + static DatabaseMatchKeyBuilderImpl instance; + static ArexMocker mocker; + + @BeforeAll + static void setUp() { + instance = new DatabaseMatchKeyBuilderImpl(); + mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("database@table@select@query"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setAttribute(ArexConstants.DB_PARAMETERS, "id=1"); + mocker.getTargetRequest().setBody("select * from table where id=?"); + mocker.setTargetResponse(new Mocker.Target()); + Mockito.mockStatic(DatabaseUtils.class); + Mockito.mockStatic(Serializer.class); + } + + @AfterAll + static void tearDown() { + instance = null; + mocker = null; + Mockito.clearAllCaches(); + } + + @Test + void isSupported() { + assertTrue(instance.isSupported(MockCategoryType.DATABASE)); + } + + @Test + void getFuzzyMatchKey() { + Mockito.when(DatabaseUtils.parseDbName(any(), any())).thenReturn("database"); + assertNotEquals(instance.getFuzzyMatchKey(mocker), 0); + Mockito.when(DatabaseUtils.parseTableNames(any())).thenReturn(Collections.singletonList("table")); + assertNotEquals(instance.getFuzzyMatchKey(mocker), 0); + } + + @Test + void getAccurateMatchKey() { + assertNotEquals(instance.getAccurateMatchKey(mocker), 0); + } + + @Test + void getEigenBody() { + assertNull(instance.getEigenBody(mocker)); + } +} \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImplTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImplTest.java new file mode 100644 index 000000000..8ecd56b24 --- /dev/null +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/DefaultMatchKeyBuilderImplTest.java @@ -0,0 +1,51 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class DefaultMatchKeyBuilderImplTest { + + static DefaultMatchKeyBuilderImpl instance; + static ArexMocker mocker; + + @BeforeAll + static void setUp() { + instance = new DefaultMatchKeyBuilderImpl(); + mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); + mocker.setOperationName("mock"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + } + + @AfterAll + static void tearDown() { + instance = null; + mocker = null; + } + @Test + void isSupported() { + assertTrue(instance.isSupported(MockCategoryType.DYNAMIC_CLASS)); + } + + @Test + void getFuzzyMatchKey() { + assertNotEquals(instance.getFuzzyMatchKey(mocker), 0); + } + + @Test + void getAccurateMatchKey() { + assertNotEquals(instance.getAccurateMatchKey(mocker), 0); + } + + @Test + void getEigenBody() { + assertNotNull(instance.getEigenBody(mocker)); + } +} \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImplTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImplTest.java new file mode 100644 index 000000000..317cff6a2 --- /dev/null +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/match/key/HttpClientMatchKeyBuilderImplTest.java @@ -0,0 +1,56 @@ +package io.arex.inst.runtime.match.key; + +import io.arex.agent.bootstrap.model.ArexMocker; +import io.arex.agent.bootstrap.model.MockCategoryType; +import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.model.ArexConstants; +import io.arex.inst.runtime.serializer.Serializer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.*; + +class HttpClientMatchKeyBuilderImplTest { + static HttpClientMatchKeyBuilderImpl instance; + static ArexMocker mocker; + + @BeforeAll + static void setUp() { + instance = new HttpClientMatchKeyBuilderImpl(); + mocker = new ArexMocker(MockCategoryType.HTTP_CLIENT); + mocker.setOperationName("http"); + mocker.setTargetRequest(new Mocker.Target()); + mocker.getTargetRequest().setBody("mock"); + mocker.setTargetResponse(new Mocker.Target()); + Mockito.mockStatic(Serializer.class); + } + + @AfterAll + static void tearDown() { + instance = null; + mocker = null; + Mockito.clearAllCaches(); + } + @Test + void isSupported() { + assertTrue(instance.isSupported(MockCategoryType.HTTP_CLIENT)); + } + + @Test + void getFuzzyMatchKey() { + assertNotEquals(instance.getFuzzyMatchKey(mocker), 0); + } + + @Test + void getAccurateMatchKey() { + assertNotEquals(instance.getAccurateMatchKey(mocker), 0); + } + + @Test + void getEigenBody() { + mocker.getTargetRequest().setAttribute(ArexConstants.HTTP_QUERY_STRING, "mock"); + assertNull(instance.getEigenBody(mocker)); + } +} \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/model/QueryAllMockerDTOTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/model/QueryAllMockerDTOTest.java index bb2ca6f7c..c22be8520 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/model/QueryAllMockerDTOTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/model/QueryAllMockerDTOTest.java @@ -51,7 +51,7 @@ void setFieldNames() { @Test void getCategoryTypes() { - assertNotNull(queryAllMockerDTO.getCategoryTypes()); + assertNull(queryAllMockerDTO.getCategoryTypes()); } @Test diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/DatabaseUtilsTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/DatabaseUtilsTest.java index a660b93af..1efa8bcf6 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/DatabaseUtilsTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/DatabaseUtilsTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; @@ -84,4 +85,25 @@ static Stream regenerateOperationNameCase() { arguments("database1", "@", "wrong sql", needRegenerateMocker, predicate1) ); } + + @ParameterizedTest + @CsvSource(value ={ + "query, database, database", + "'', null, ''", + "query, null, query", + "database@table, null, database", + }, nullValues={"null"}) + void parseDbName(String source, String strip, String expect) { + assertEquals(expect, DatabaseUtils.parseDbName(source, strip)); + } + + @ParameterizedTest + @CsvSource(value ={ + "''", + "query", + "table2@select@operation1;db2@table3", + }, nullValues={"null"}) + void parseDbName(String source) { + assertNotNull(DatabaseUtils.parseTableNames(source)); + } } \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockManagerTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockManagerTest.java new file mode 100644 index 000000000..8a51d2dcb --- /dev/null +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockManagerTest.java @@ -0,0 +1,42 @@ +package io.arex.inst.runtime.util; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.*; + +class MockManagerTest { + + @BeforeAll + static void setUp() { + Mockito.mockStatic(MockUtils.class); + Mockito.mockStatic(MergeRecordUtil.class); + } + + @AfterAll + static void tearDown() { + Mockito.clearAllCaches(); + } + + @Test + void mergeRecord() { + assertDoesNotThrow(() -> MockManager.mergeRecord(null)); + } + + @Test + void recordRemain() { + assertDoesNotThrow(() -> MockManager.recordRemain(null)); + } + + @Test + void executeRecord() { + assertDoesNotThrow(() -> MockManager.executeRecord(null)); + } + + @Test + void saveReplayRemainCompareRelation() { + assertDoesNotThrow(() -> MockManager.saveReplayRemainCompareRelation(null)); + } +} \ No newline at end of file diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockUtilsTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockUtilsTest.java index d848fff69..65e1bacdc 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockUtilsTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/MockUtilsTest.java @@ -6,13 +6,12 @@ import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockCategoryType; -import io.arex.agent.bootstrap.model.Mocker; -import io.arex.inst.runtime.config.Config; import io.arex.inst.runtime.config.ConfigBuilder; import io.arex.inst.runtime.context.ArexContext; import io.arex.inst.runtime.context.ContextManager; import io.arex.inst.runtime.listener.EventProcessorTest.TestGsonSerializer; import io.arex.inst.runtime.listener.EventProcessorTest.TestJacksonSerializable; +import io.arex.inst.runtime.log.LogManager; import io.arex.inst.runtime.match.ReplayMatcher; import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.QueryAllMockerDTO; @@ -38,12 +37,6 @@ static void setUp() { Mockito.mockStatic(CaseManager.class); configBuilder = ConfigBuilder.create("test"); - - Mockito.mockStatic(Config.class); - Config config = Mockito.mock(Config.class); - Mockito.when(Config.get()).thenReturn(config); - Mockito.when(config.isEnableDebug()).thenReturn(true); - dataCollector = Mockito.mock(DataCollector.class); DataService.setDataCollector(Collections.singletonList(dataCollector)); @@ -99,8 +92,7 @@ void replayMocker() { Mockito.when(CaseManager.isInvalidCase("mock-replay-id")).thenReturn(false); Mockito.when(ContextManager.currentContext()).thenReturn(ArexContext.of("mock-trace-id", "mock-replay-id")); dynamicClass = MockUtils.createDynamicClass("test", "test"); - Object actualResult = MockUtils.replayBody(dynamicClass); - assertEquals(1693194255518L, actualResult); + assertNull(MockUtils.replayBody(dynamicClass)); // invalid case Mockito.when(CaseManager.isInvalidCase("mock-replay-id")).thenReturn(true); @@ -111,14 +103,26 @@ void replayMocker() { Mockito.when(ContextManager.currentContext()).thenReturn(ArexContext.of("mock-trace-id", null)); assertNull(MockUtils.replayBody(dynamicClass)); + configBuilder.enableDebug(true); + configBuilder.build(); + // null replayId and is config file ArexMocker configFile = MockUtils.createConfigFile("test"); - assertNotNull(MockUtils.replayBody(configFile)); + assertNull(MockUtils.replayBody(configFile)); // merge case configFile.setNeedMerge(true); Mockito.when(ReplayMatcher.match(any(), any())).thenReturn(configFile); assertNull(MockUtils.replayBody(configFile)); + + Mockito.when(dataCollector.query(anyString(), any())).thenReturn(""); + assertNull(MockUtils.replayBody(configFile)); + + configBuilder.enableDebug(false); + configBuilder.build(); + + Mockito.when(dataCollector.query(anyString(), any())).thenReturn(responseJson); + assertNull(MockUtils.replayBody(configFile)); } @Test @@ -190,31 +194,22 @@ void createMocker() { } @Test - void methodSignatureHash() { - ArexMocker mocker = new ArexMocker(); - mocker.setTargetRequest(new Mocker.Target()); - mocker.getTargetRequest().setBody("mock"); - assertTrue(MockUtils.methodSignatureHash(mocker) > 0); - } + void queryMockers() { + Mockito.mockStatic(Serializer.class); + QueryAllMockerDTO requestMocker = new QueryAllMockerDTO(); + assertNull(MockUtils.queryMockers(requestMocker)); - @Test - void methodRequestTypeHash() { - ArexMocker mocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); - mocker.setTargetRequest(new Mocker.Target()); - mocker.getTargetRequest().setBody("mock"); - assertTrue(MockUtils.methodRequestTypeHash(mocker) > 0); + configBuilder.enableDebug(true); + configBuilder.build(); + Mockito.when(dataCollector.queryAll(any())).thenReturn("mock"); + assertNull(MockUtils.queryMockers(requestMocker)); } @Test - void queryMockers() { - QueryAllMockerDTO requestMocker = new QueryAllMockerDTO(); - requestMocker.setRecordId("mock"); - requestMocker.setReplayId("mock"); - MockUtils.queryMockers(requestMocker); - - String responseJson = "{\"categoryType\":{\"name\":\"DynamicClass\"},\"recordId\":\"mock\"," + - "\"operationName\":\"java.lang.System.currentTimeMillis\"}"; - Mockito.when(dataCollector.queryAll(any())).thenReturn(responseJson); - MockUtils.queryMockers(requestMocker); + void saveReplayCompareResult() { + Mockito.mockStatic(LogManager.class); + configBuilder.enableDebug(true); + configBuilder.build(); + assertDoesNotThrow(() -> MockUtils.saveReplayCompareResult(null, new ArrayList<>())); } } diff --git a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/ReplayUtilTest.java b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/ReplayUtilTest.java index 0f9e4411c..43f760092 100644 --- a/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/ReplayUtilTest.java +++ b/arex-instrumentation-api/src/test/java/io/arex/inst/runtime/util/ReplayUtilTest.java @@ -3,100 +3,162 @@ import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockCategoryType; import io.arex.agent.bootstrap.model.Mocker; -import io.arex.agent.bootstrap.util.CollectionUtil; +import io.arex.agent.thirdparty.util.CompressUtil; import io.arex.inst.runtime.config.Config; import io.arex.inst.runtime.context.ArexContext; import io.arex.inst.runtime.context.ContextManager; import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.MergeDTO; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; class ReplayUtilTest { static ArexMocker requestMocker; + static ArexContext context; @BeforeAll static void setUp() { Mockito.mockStatic(MockUtils.class); Mockito.mockStatic(ContextManager.class); + Mockito.when(ContextManager.needReplay()).thenReturn(true); + context = Mockito.mock(ArexContext.class); + Mockito.when(ContextManager.currentContext()).thenReturn(context); Mockito.mockStatic(Config.class); - Mockito.when(Config.get()).thenReturn(Mockito.mock(Config.class)); + Config config = Mockito.mock(Config.class); + Mockito.when(Config.get()).thenReturn(config); + Mockito.when(config.isEnableDebug()).thenReturn(true); requestMocker = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); requestMocker.setOperationName("mock"); requestMocker.setTargetRequest(new Mocker.Target()); requestMocker.setTargetResponse(new Mocker.Target()); Mockito.when(MockUtils.create(any(), any())).thenReturn(requestMocker); Mockito.mockStatic(Serializer.class); + Mockito.mockStatic(CompressUtil.class); } @AfterAll static void tearDown() { requestMocker = null; + context = null; Mockito.clearAllCaches(); } @ParameterizedTest - @MethodSource("replayAllMockerCase") - void replayAllMocker(Runnable mocker) { + @MethodSource("queryMockersCase") + void queryMockers(Runnable mocker) { mocker.run(); assertDoesNotThrow(ReplayUtil::queryMockers); } - static Stream replayAllMockerCase() { - ArexMocker recordMocker1 = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); - recordMocker1.setOperationName(ArexConstants.MERGE_RECORD_NAME); - recordMocker1.setRecordId("mock"); - recordMocker1.setReplayId("mock"); - recordMocker1.setTargetRequest(new Mocker.Target()); - recordMocker1.setTargetResponse(new Mocker.Target()); - recordMocker1.getTargetResponse().setBody("mock"); - recordMocker1.setCreationTime(System.currentTimeMillis()); - ArexMocker recordMocker2 = new ArexMocker(MockCategoryType.DYNAMIC_CLASS); - recordMocker2.setOperationName(ArexConstants.MERGE_RECORD_NAME); - recordMocker2.setRecordId("mock"); - recordMocker2.setReplayId("mock"); - recordMocker2.setTargetRequest(new Mocker.Target()); - recordMocker2.setTargetResponse(new Mocker.Target()); - recordMocker2.getTargetResponse().setBody("mock"); - recordMocker2.setCreationTime(System.currentTimeMillis() + 1); - - MergeDTO mergeDTO = new MergeDTO(); + static Stream queryMockersCase() { + List recordMockerList = new ArrayList<>(); + recordMockerList.add(requestMocker); Runnable emptyMocker = () -> {}; Runnable mocker1 = () -> { - Mockito.when(ContextManager.needReplay()).thenReturn(true); - ArexContext context = Mockito.mock(ArexContext.class); - Mockito.when(ContextManager.currentContext()).thenReturn(context); Map> cachedReplayResultMap = new HashMap<>(); Mockito.when(context.getCachedReplayResultMap()).thenReturn(cachedReplayResultMap); - Mockito.when(MockUtils.queryMockers(any())).thenReturn(CollectionUtil.newArrayList(recordMocker1, recordMocker2)); - Mockito.when(Serializer.deserialize("mock", ArexConstants.MERGE_TYPE)) - .thenReturn(CollectionUtil.newArrayList(mergeDTO)); }; Runnable mocker2 = () -> { Mockito.when(MockUtils.checkResponseMocker(any())).thenReturn(true); - requestMocker.getTargetResponse().setBody("mock"); - Mockito.when(MockUtils.executeReplay(any(), any())).thenReturn(requestMocker); - + requestMocker.setRequest("mock"); + requestMocker.setResponse("mock"); + ArexMocker databaseMocker = new ArexMocker(MockCategoryType.DATABASE); + databaseMocker.setOperationName("databse@table@select@query"); + databaseMocker.setTargetRequest(new Mocker.Target()); + databaseMocker.getTargetRequest().setBody("mock"); + databaseMocker.setTargetResponse(new Mocker.Target()); + recordMockerList.add(databaseMocker); + ArexMocker databaseMocker2 = new ArexMocker(MockCategoryType.DATABASE); + databaseMocker2.setOperationName("databse@table@select@query"); + databaseMocker2.setTargetRequest(new Mocker.Target()); + databaseMocker2.getTargetRequest().setBody("mock"); + databaseMocker2.setTargetResponse(new Mocker.Target()); + recordMockerList.add(databaseMocker2); + ArexMocker databaseMocker3 = new ArexMocker(MockCategoryType.DATABASE); + databaseMocker3.setOperationName("databse@table@select@query"); + databaseMocker3.setTargetRequest(new Mocker.Target()); + databaseMocker3.getTargetRequest().setBody("mock"); + databaseMocker3.setTargetResponse(new Mocker.Target()); + databaseMocker3.setCreationTime(System.currentTimeMillis() - 1000); + recordMockerList.add(databaseMocker3); + Mockito.when(MockUtils.queryMockers(any())).thenReturn(recordMockerList); + Mockito.when(Serializer.deserialize(any(), eq(ArexConstants.MOCKER_TARGET_TYPE))).thenReturn(new Mocker.Target()); + }; + Runnable mocker3 = () -> { + requestMocker.setOperationName("arex.mergeRecord"); + Mockito.when(Serializer.deserialize(any(), eq(ArexConstants.MERGE_TYPE))).thenReturn(null); + }; + List mergeReplayList = new ArrayList<>(); + MergeDTO mergeDTO = new MergeDTO(); + mergeReplayList.add(mergeDTO); + Runnable mocker4 = () -> { + Mockito.when(Serializer.deserialize(any(), eq(ArexConstants.MERGE_TYPE))).thenReturn(mergeReplayList); + }; + Runnable mocker5 = () -> { mergeDTO.setCategory(MockCategoryType.DYNAMIC_CLASS.getName()); }; return Stream.of( arguments(emptyMocker), arguments(mocker1), - arguments(mocker2) + arguments(mocker2), + arguments(mocker3), + arguments(mocker4), + arguments(mocker5) ); } + + @Test + void saveReplayCompareResult() { + assertDoesNotThrow(ReplayUtil::saveReplayCompareResult); + // replayCompareResultQueue is empty + Mockito.when(context.isReplay()).thenReturn(true); + LinkedBlockingQueue replayCompareResultQueue = new LinkedBlockingQueue<>(); + Mockito.when(context.getReplayCompareResultQueue()).thenReturn(replayCompareResultQueue); + assertDoesNotThrow(ReplayUtil::saveReplayCompareResult); + // saveReplayCompareResult + replayCompareResultQueue.offer(new ReplayCompareResultDTO()); + Mockito.when(context.getReplayCompareResultQueue()).thenReturn(replayCompareResultQueue); + Map> cachedReplayResultMap = new HashMap<>(); + cachedReplayResultMap.put(1, Collections.singletonList(requestMocker)); + Mockito.when(context.getCachedReplayResultMap()).thenReturn(cachedReplayResultMap); + assertDoesNotThrow(ReplayUtil::saveReplayCompareResult); + } + + @Test + void saveRemainCompareResult() { + assertDoesNotThrow(() -> ReplayUtil.saveRemainCompareResult(null)); + + Mockito.when(context.getReplayCompareResultQueue()).thenReturn(new LinkedBlockingQueue<>()); + Map> cachedReplayResultMap = new HashMap<>(); + ArexMocker databaseMocker = new ArexMocker(MockCategoryType.DATABASE); + databaseMocker.setOperationName("databse@table@select@query"); + databaseMocker.setTargetRequest(new Mocker.Target()); + databaseMocker.getTargetRequest().setBody("mock"); + databaseMocker.setTargetResponse(new Mocker.Target()); + cachedReplayResultMap.put(1, Collections.singletonList(databaseMocker)); + Mockito.when(context.getCachedReplayResultMap()).thenReturn(cachedReplayResultMap); + assertDoesNotThrow(() -> ReplayUtil.saveRemainCompareResult(context)); + } } diff --git a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/config/ConfigManager.java b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/config/ConfigManager.java index 4bc40322f..12d4f7a16 100644 --- a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/config/ConfigManager.java +++ b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/config/ConfigManager.java @@ -9,10 +9,14 @@ import io.arex.foundation.model.ConfigQueryResponse.DynamicClassConfiguration; import io.arex.foundation.model.ConfigQueryResponse.ResponseBody; import io.arex.foundation.model.ConfigQueryResponse.ServiceCollectConfig; +import io.arex.foundation.model.ConfigQueryResponse.CompareConfiguration; +import io.arex.foundation.model.ConfigQueryResponse.ConfigComparisonExclusions; import io.arex.agent.bootstrap.util.CollectionUtil; import io.arex.inst.runtime.config.Config; import io.arex.inst.runtime.config.ConfigBuilder; import io.arex.inst.runtime.config.listener.ConfigListener; +import io.arex.inst.runtime.model.CompareConfigurationEntity; +import io.arex.inst.runtime.model.CompareConfigurationEntity.ConfigComparisonExclusionsEntity; import io.arex.inst.runtime.model.DynamicClassEntity; import io.arex.inst.runtime.model.DynamicClassStatusEnum; import io.arex.agent.bootstrap.util.ServiceLoader; @@ -64,6 +68,7 @@ public class ConfigManager { private List listeners = new ArrayList<>(); private Map extendField; private int bufferSize; + private CompareConfigurationEntity compareConfigurationEntity; private ConfigManager() { init(); @@ -356,6 +361,7 @@ public void updateConfigFromService(ResponseBody serviceConfig) { setAgentEnabled(serviceConfig.isAgentEnabled()); setExtendField(serviceConfig.getExtendField()); setMessage(serviceConfig.getMessage()); + setCompareConfiguration(serviceConfig.getCompareConfiguration()); updateRuntimeConfig(); } @@ -385,6 +391,7 @@ private void updateRuntimeConfig() { .excludeServiceOperations(getExcludeServiceOperations()) .dubboStreamReplayThreshold(getDubboStreamReplayThreshold()) .recordRate(getRecordRate()) + .compareConfiguration(getCompareConfiguration()) .build(); publish(Config.get()); } @@ -630,6 +637,31 @@ public void setBufferSize(String bufferSize) { System.setProperty(BUFFER_SIZE, bufferSize); } + public void setCompareConfiguration(CompareConfiguration compareConfiguration) { + if (compareConfiguration == null) { + return; + } + List comparisonExclusions = compareConfiguration.getComparisonExclusions(); + List comparisonExclusionsEntities = new ArrayList<>(); + if (CollectionUtil.isNotEmpty(comparisonExclusions)) { + for (ConfigComparisonExclusions comparisonExclusion : comparisonExclusions) { + ConfigComparisonExclusionsEntity exclusionsEntity = new ConfigComparisonExclusionsEntity(); + exclusionsEntity.setCategoryType(comparisonExclusion.getCategoryType()); + exclusionsEntity.setOperationName(comparisonExclusion.getOperationName()); + exclusionsEntity.setExclusionList(comparisonExclusion.getExclusionList()); + comparisonExclusionsEntities.add(exclusionsEntity); + } + } + compareConfigurationEntity = new CompareConfigurationEntity(); + compareConfigurationEntity.setComparisonExclusions(comparisonExclusionsEntities); + compareConfigurationEntity.setGlobalExclusionList(compareConfiguration.getGlobalExclusionList()); + compareConfigurationEntity.setIgnoreNodeSet(compareConfiguration.getIgnoreNodeSet()); + } + + public CompareConfigurationEntity getCompareConfiguration() { + return compareConfigurationEntity; + } + @Override public String toString() { return "ConfigManager{" + diff --git a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/model/ConfigQueryResponse.java b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/model/ConfigQueryResponse.java index c4a54c318..075b50f53 100644 --- a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/model/ConfigQueryResponse.java +++ b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/model/ConfigQueryResponse.java @@ -63,6 +63,7 @@ public static class ResponseBody { private List dynamicClassConfigurationList; private boolean agentEnabled; private Map extendField; + private CompareConfiguration compareConfiguration; public ServiceCollectConfig getServiceCollectConfiguration() { return serviceCollectConfiguration; @@ -113,6 +114,14 @@ public Map getExtendField() { public void setExtendField(Map extendField) { this.extendField = extendField; } + + public CompareConfiguration getCompareConfiguration() { + return compareConfiguration; + } + + public void setCompareConfiguration(CompareConfiguration compareConfiguration) { + this.compareConfiguration = compareConfiguration; + } } public static class ServiceCollectConfig { @@ -221,4 +230,66 @@ public void setKeyFormula(String keyFormula) { this.keyFormula = keyFormula; } } + + public static class CompareConfiguration { + private List comparisonExclusions; + + private Set> globalExclusionList; + + private Set ignoreNodeSet; + + public List getComparisonExclusions() { + return comparisonExclusions; + } + + public void setComparisonExclusions(List comparisonExclusions) { + this.comparisonExclusions = comparisonExclusions; + } + + public Set> getGlobalExclusionList() { + return globalExclusionList; + } + + public void setGlobalExclusionList(Set> globalExclusionList) { + this.globalExclusionList = globalExclusionList; + } + + public Set getIgnoreNodeSet() { + return ignoreNodeSet; + } + + public void setIgnoreNodeSet(Set ignoreNodeSet) { + this.ignoreNodeSet = ignoreNodeSet; + } + } + + public static class ConfigComparisonExclusions { + private String operationName; + private String categoryType; + private Set> exclusionList; + + public String getOperationName() { + return operationName; + } + + public void setOperationName(String operationName) { + this.operationName = operationName; + } + + public String getCategoryType() { + return categoryType; + } + + public void setCategoryType(String categoryType) { + this.categoryType = categoryType; + } + + public Set> getExclusionList() { + return exclusionList; + } + + public void setExclusionList(Set> exclusionList) { + this.exclusionList = exclusionList; + } + } } diff --git a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/services/DataCollectorService.java b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/services/DataCollectorService.java index 90b3d9258..f21f09863 100644 --- a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/services/DataCollectorService.java +++ b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/services/DataCollectorService.java @@ -4,6 +4,7 @@ import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockStrategyEnum; import io.arex.agent.bootstrap.model.Mocker; +import io.arex.agent.bootstrap.util.CollectionUtil; import io.arex.agent.bootstrap.util.MapUtils; import io.arex.agent.bootstrap.util.StringUtil; import io.arex.foundation.config.ConfigManager; @@ -15,7 +16,9 @@ import io.arex.foundation.model.HttpClientResponse; import io.arex.foundation.util.httpclient.async.ThreadFactoryImpl; import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.QueryAllMockerDTO; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; import io.arex.inst.runtime.util.CaseManager; import io.arex.inst.runtime.service.DataCollector; @@ -47,6 +50,7 @@ public class DataCollectorService implements DataCollector { private static String saveApiUrl; private static String invalidCaseApiUrl; private static String queryAllApiUrl; + private static String batchSaveReplayResult; static { initServiceHost(); @@ -134,7 +138,7 @@ void saveData(DataEntity entity) { return; } AsyncHttpClientUtil.postAsyncWithZstdJson(saveApiUrl, entity.getPostData(), null) - .whenComplete(saveMockDataConsumer(entity)); + .whenComplete(saveMockDataConsumer(entity)); } /** @@ -187,7 +191,8 @@ private static void initServiceHost() { queryApiUrl = String.format("http://%s/api/storage/record/query", storeServiceHost); saveApiUrl = String.format("http://%s/api/storage/record/batchSaveMockers", storeServiceHost); invalidCaseApiUrl = String.format("http://%s/api/storage/record/invalidCase", storeServiceHost); - queryAllApiUrl = String.format("http://%s/api/storage/record/queryMockers", storeServiceHost); + queryAllApiUrl = String.format("http://%s/api/storage/record/batchQueryMockers", storeServiceHost); + batchSaveReplayResult = String.format("http://%s/api/storage/record/batchSaveReplayResult", storeServiceHost); } @Override @@ -208,7 +213,7 @@ private BiFunction queryAllMo QueryAllMockerDTO mocker = Serializer.deserialize(postData, QueryAllMockerDTO.class); if (mocker != null) { CaseManager.invalid(mocker.getRecordId(), mocker.getReplayId(), - "queryAllMockers", DecelerateReasonEnum.SERVICE_EXCEPTION.getValue()); + "batchQueryMockers", DecelerateReasonEnum.SERVICE_EXCEPTION.getValue()); } return null; } @@ -216,6 +221,27 @@ private BiFunction queryAllMo }; } + @Override + public void saveReplayCompareResult(String postData) { + AsyncHttpClientUtil.postAsyncWithZstdJson(batchSaveReplayResult, postData, null) + .whenComplete(saveReplayCompareConsumer(postData)); + } + + private BiConsumer saveReplayCompareConsumer(String postData) { + return (response, throwable) -> { + if (Objects.nonNull(throwable)) { + List replayCompareList = Serializer.deserialize(postData, ArexConstants.REPLAY_COMPARE_TYPE); + if (CollectionUtil.isNotEmpty(replayCompareList)) { + CaseManager.invalid(replayCompareList.get(0).getRecordId(), replayCompareList.get(0).getReplayId(), + "saveReplayCompareResult", DecelerateReasonEnum.SERVICE_EXCEPTION.getValue()); + } + LogManager.warn("saveReplayCompareResult", throwable); + } else { + LogManager.info("saveReplayCompareResult.response", StringUtil.format("response: %s", Serializer.serialize(response))); + } + }; + } + @Override public int order() { return 0; diff --git a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/AsyncHttpClientUtil.java b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/AsyncHttpClientUtil.java index a4a6b9691..5707c5aa9 100644 --- a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/AsyncHttpClientUtil.java +++ b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/AsyncHttpClientUtil.java @@ -3,9 +3,9 @@ import io.arex.agent.bootstrap.constants.ConfigConstants; import io.arex.agent.bootstrap.util.MapUtils; import io.arex.agent.bootstrap.util.StringUtil; +import io.arex.agent.thirdparty.util.CompressUtil; import io.arex.foundation.config.ConfigManager; import io.arex.foundation.model.HttpClientResponse; -import io.arex.foundation.util.CompressUtil; import io.arex.foundation.util.httpclient.async.AutoCleanedPoolingNHttpClientConnectionManager; import io.arex.foundation.util.httpclient.async.ThreadFactoryImpl; import io.arex.inst.runtime.log.LogManager; @@ -41,7 +41,7 @@ public class AsyncHttpClientUtil { private static final long RECORD_BODY_MAX_LIMIT_5MB = 5 * 1024L * 1024L; private static CloseableHttpAsyncClient asyncClient; private static final CompletableFuture EMPTY_RESPONSE = CompletableFuture.completedFuture( - HttpClientResponse.emptyResponse()); + HttpClientResponse.emptyResponse()); private AsyncHttpClientUtil() { } @@ -56,7 +56,7 @@ private AsyncHttpClientUtil() { } public static CompletableFuture postAsyncWithJson(String uri, String postData, - Map requestHeaders) { + Map requestHeaders) { HttpEntity httpEntity = new ByteArrayEntity(postData.getBytes(StandardCharsets.UTF_8)); if (requestHeaders == null) { @@ -68,7 +68,7 @@ public static CompletableFuture postAsyncWithJson(String uri } public static CompletableFuture postAsyncWithZstdJson(String uri, String postData, - Map requestHeaders) { + Map requestHeaders) { HttpEntity httpEntity = new ByteArrayEntity(CompressUtil.zstdCompress(postData, StandardCharsets.UTF_8)); if (requestHeaders == null) { @@ -80,7 +80,7 @@ public static CompletableFuture postAsyncWithZstdJson(String } public static CompletableFuture executeAsync(String uri, HttpEntity httpEntity, - Map requestHeaders, HttpClientResponseHandler responseHandler) { + Map requestHeaders, HttpClientResponseHandler responseHandler) { if (httpEntity.getContentLength() > RECORD_BODY_MAX_LIMIT_5MB || httpEntity.getContentLength() < 0) { LogManager.warn("executeAsync", "do not record, the size is larger than 5MB."); return EMPTY_RESPONSE; @@ -95,7 +95,7 @@ public static CompletableFuture executeAsync(String uri, Htt private static HttpUriRequest createHttpPost(String uri, HttpEntity httpEntity, Map requestHeaders) { HttpPost httpPost = prepareHttpRequest(uri, ClientConfig.DEFAULT_CONNECT_TIMEOUT, - ClientConfig.DEFAULT_SOCKET_TIMEOUT); + ClientConfig.DEFAULT_SOCKET_TIMEOUT); httpPost.setEntity(httpEntity); if (MapUtils.isNotEmpty(requestHeaders)) { @@ -121,23 +121,23 @@ private static HttpPost prepareHttpRequest(String uri, int connectTimeout, int s private static CloseableHttpAsyncClient createAsyncClient() { RequestConfig defaultRequestConfig = - createRequestConfig(ClientConfig.DEFAULT_CONNECT_TIMEOUT, ClientConfig.DEFAULT_SOCKET_TIMEOUT); + createRequestConfig(ClientConfig.DEFAULT_CONNECT_TIMEOUT, ClientConfig.DEFAULT_SOCKET_TIMEOUT); AutoCleanedPoolingNHttpClientConnectionManager connectionManager = - AutoCleanedPoolingNHttpClientConnectionManager.createDefault(); + AutoCleanedPoolingNHttpClientConnectionManager.createDefault(); asyncClient = HttpAsyncClients.custom() - .setThreadFactory(new ThreadFactoryImpl("arex-async-http-client")) - .setDefaultRequestConfig(defaultRequestConfig) - .setConnectionManager(connectionManager).build(); + .setThreadFactory(new ThreadFactoryImpl("arex-async-http-client")) + .setDefaultRequestConfig(defaultRequestConfig) + .setConnectionManager(connectionManager).build(); return asyncClient; } private static RequestConfig createRequestConfig(int connectTimeout, int socketTimeout) { return RequestConfig.custom() - .setConnectionRequestTimeout(ClientConfig.DEFAULT_CONNECTION_REQUEST_TIMEOUT) - .setConnectTimeout(connectTimeout) - .setSocketTimeout(socketTimeout).build(); + .setConnectionRequestTimeout(ClientConfig.DEFAULT_CONNECTION_REQUEST_TIMEOUT) + .setConnectTimeout(connectTimeout) + .setSocketTimeout(socketTimeout).build(); } static class ClientConfig { @@ -150,12 +150,12 @@ static class ClientConfig { private static final Header HEADER_ACCEPT = new BasicHeader(HttpHeaders.ACCEPT, "*/*"); private static final Header HEADER_USER_AGENT = new BasicHeader(HttpHeaders.USER_AGENT, - String.format("arex-async-http-client-%s", ConfigManager.INSTANCE.getAgentVersion())); + String.format("arex-async-http-client-%s", ConfigManager.INSTANCE.getAgentVersion())); private static final Header HEADER_API_TOKEN = new BasicHeader("arex-api-token", - System.getProperty(ConfigConstants.API_TOKEN, StringUtil.EMPTY)); + System.getProperty(ConfigConstants.API_TOKEN, StringUtil.EMPTY)); private static final Header HEADER_SERVICE_NAME = new BasicHeader("arex-service-name", - System.getProperty(ConfigConstants.SERVICE_NAME, StringUtil.EMPTY)); + System.getProperty(ConfigConstants.SERVICE_NAME, StringUtil.EMPTY)); private static final Header HEADER_AGENT_VERSION = new BasicHeader("arex-agent-version", - System.getProperty(ConfigConstants.AGENT_VERSION, StringUtil.EMPTY)); + System.getProperty(ConfigConstants.AGENT_VERSION, StringUtil.EMPTY)); } } diff --git a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/HttpClientResponseHandler.java b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/HttpClientResponseHandler.java index 030283749..756ae877f 100644 --- a/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/HttpClientResponseHandler.java +++ b/arex-instrumentation-foundation/src/main/java/io/arex/foundation/util/httpclient/HttpClientResponseHandler.java @@ -1,8 +1,9 @@ package io.arex.foundation.util.httpclient; -import io.arex.foundation.util.CompressUtil; import java.io.IOException; import java.nio.charset.StandardCharsets; + +import io.arex.agent.thirdparty.util.CompressUtil; import org.apache.http.HttpEntity; import org.apache.http.util.EntityUtils; diff --git a/arex-instrumentation-foundation/src/test/java/io/arex/foundation/config/ConfigManagerTest.java b/arex-instrumentation-foundation/src/test/java/io/arex/foundation/config/ConfigManagerTest.java index 72499d737..5bfb414e4 100644 --- a/arex-instrumentation-foundation/src/test/java/io/arex/foundation/config/ConfigManagerTest.java +++ b/arex-instrumentation-foundation/src/test/java/io/arex/foundation/config/ConfigManagerTest.java @@ -8,7 +8,6 @@ import io.arex.inst.runtime.model.DynamicClassEntity; import io.arex.inst.runtime.model.DynamicClassStatusEnum; import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.time.LocalDate; import java.time.LocalTime; @@ -316,4 +315,15 @@ void appendCoveragePackages() throws Exception { appendCoveragePackages.invoke(configManager, "com.a.b"); assertEquals("com.a.b", System.getProperty(ConfigConstants.COVERAGE_PACKAGES)); } + + @Test + void setCompareConfiguration() { + assertDoesNotThrow(() -> configManager.setCompareConfiguration(null)); + ConfigQueryResponse.CompareConfiguration compareConfiguration = new ConfigQueryResponse.CompareConfiguration(); + List comparisonExclusions = new ArrayList<>(); + ConfigQueryResponse.ConfigComparisonExclusions exclusion = new ConfigQueryResponse.ConfigComparisonExclusions(); + comparisonExclusions.add(exclusion); + compareConfiguration.setComparisonExclusions(comparisonExclusions); + assertDoesNotThrow(() -> configManager.setCompareConfiguration(compareConfiguration)); + } } diff --git a/arex-instrumentation-foundation/src/test/java/io/arex/foundation/services/DataCollectorServiceTest.java b/arex-instrumentation-foundation/src/test/java/io/arex/foundation/services/DataCollectorServiceTest.java index a124a8c15..faab00dac 100644 --- a/arex-instrumentation-foundation/src/test/java/io/arex/foundation/services/DataCollectorServiceTest.java +++ b/arex-instrumentation-foundation/src/test/java/io/arex/foundation/services/DataCollectorServiceTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import io.arex.agent.bootstrap.model.ArexMocker; import io.arex.agent.bootstrap.model.MockStrategyEnum; @@ -19,7 +20,10 @@ import java.util.Collections; import java.util.concurrent.CompletableFuture; +import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.model.QueryAllMockerDTO; +import io.arex.inst.runtime.model.ReplayCompareResultDTO; import io.arex.inst.runtime.serializer.Serializer; import io.arex.inst.runtime.util.CaseManager; import org.junit.jupiter.api.AfterAll; @@ -37,6 +41,7 @@ static void setUp() { Mockito.mockStatic(ContextManager.class); Mockito.mockStatic(Serializer.class); caseManagerMocked = Mockito.mockStatic(CaseManager.class); + Mockito.mockStatic(LogManager.class); } @AfterAll @@ -137,4 +142,18 @@ void queryAll() { void order() { assertEquals(0, DataCollectorService.INSTANCE.order()); } + + @Test + void saveReplayCompareResult() { + Mockito.when(AsyncHttpClientUtil.postAsyncWithZstdJson(anyString(), anyString(), any())) + .thenReturn(CompletableFuture.completedFuture(null)); + assertDoesNotThrow(() -> DataCollectorService.INSTANCE.saveReplayCompareResult("mock")); + // exception + CompletableFuture mockException = new CompletableFuture<>(); + mockException.completeExceptionally(new RuntimeException("mock exception")); + Mockito.when(AsyncHttpClientUtil.postAsyncWithZstdJson(anyString(), any(), any())).thenReturn(mockException); + Mockito.when(Serializer.deserialize(any(), eq(ArexConstants.REPLAY_COMPARE_TYPE))). + thenReturn(Collections.singletonList(new ReplayCompareResultDTO())); + assertDoesNotThrow(() -> DataCollectorService.INSTANCE.saveReplayCompareResult("mock")); + } } diff --git a/arex-instrumentation/database/arex-database-common/src/main/java/io/arex/inst/database/common/DatabaseExtractor.java b/arex-instrumentation/database/arex-database-common/src/main/java/io/arex/inst/database/common/DatabaseExtractor.java index bedeeb76b..49d15fc8f 100644 --- a/arex-instrumentation/database/arex-database-common/src/main/java/io/arex/inst/database/common/DatabaseExtractor.java +++ b/arex-instrumentation/database/arex-database-common/src/main/java/io/arex/inst/database/common/DatabaseExtractor.java @@ -93,11 +93,13 @@ public MockResult replay() { } public MockResult replay(String serializer) { + String operationName = DatabaseUtils.regenerateOperationName(this.dbName, this.methodName, this.sql); String serviceKey = StringUtil.defaultIfEmpty(this.dbName, ArexConstants.DATABASE); - String operationKey = DatabaseUtils.regenerateOperationName(this.dbName, this.methodName, this.sql); - boolean ignoreMockResult = IgnoreUtils.ignoreMockResult(serviceKey, operationKey); - - Mocker replayMocker = MockUtils.replayMocker(makeMocker(null, serializer)); + boolean ignoreMockResult = IgnoreUtils.ignoreMockResult(serviceKey, operationName); + Mocker mocker = makeMocker(null, serializer); + // only parse operationName on replay (record parse sql on storage service, avoid consuming application cpu during record) + mocker.setOperationName(operationName); + Mocker replayMocker = MockUtils.replayMocker(mocker); Object replayResult = null; if (MockUtils.checkResponseMocker(replayMocker)) { if (ArexConstants.JACKSON_SERIALIZER_WITH_TYPE.equals(replayMocker.getTargetResponse().getAttribute(ArexConstants.AREX_SERIALIZER))) { diff --git a/arex-instrumentation/dubbo/arex-dubbo-common/src/main/java/io/arex/inst/dubbo/common/DubboExtractor.java b/arex-instrumentation/dubbo/arex-dubbo-common/src/main/java/io/arex/inst/dubbo/common/DubboExtractor.java index 8829f04af..f739d3123 100644 --- a/arex-instrumentation/dubbo/arex-dubbo-common/src/main/java/io/arex/inst/dubbo/common/DubboExtractor.java +++ b/arex-instrumentation/dubbo/arex-dubbo-common/src/main/java/io/arex/inst/dubbo/common/DubboExtractor.java @@ -61,5 +61,6 @@ protected static void setResponseHeader(BiConsumer consumer) { protected static void addAttachmentsToContext(AbstractAdapter adapter) { ContextManager.setAttachment(ArexConstants.FORCE_RECORD, adapter.forceRecord()); ContextManager.setAttachment(ArexConstants.SCHEDULE_REPLAY, adapter.getAttachment(ArexConstants.SCHEDULE_REPLAY)); + ContextManager.setAttachment(ArexConstants.REPLAY_END_FLAG, adapter.getAttachment(ArexConstants.REPLAY_END_FLAG)); } } diff --git a/arex-instrumentation/dynamic/arex-dynamic-common/src/main/java/io/arex/inst/dynamic/common/DynamicClassExtractor.java b/arex-instrumentation/dynamic/arex-dynamic-common/src/main/java/io/arex/inst/dynamic/common/DynamicClassExtractor.java index 3276d602f..ef1c15e98 100644 --- a/arex-instrumentation/dynamic/arex-dynamic-common/src/main/java/io/arex/inst/dynamic/common/DynamicClassExtractor.java +++ b/arex-instrumentation/dynamic/arex-dynamic-common/src/main/java/io/arex/inst/dynamic/common/DynamicClassExtractor.java @@ -397,9 +397,9 @@ private String serialize(Object object, String serializer) { private int buildNoArgMethodSignatureHash(boolean isNeedResult) { if (isNeedResult) { - return StringUtil.encodeAndHash(String.format("%s_%s_%s", this.clazzName, this.methodName, getSerializedResult())); + return StringUtil.encodeAndHash(this.clazzName, this.methodName, getSerializedResult()); } - return StringUtil.encodeAndHash(String.format("%s_%s_%s", this.clazzName, this.methodName, null)); + return StringUtil.encodeAndHash(this.clazzName, this.methodName, null); } /** diff --git a/arex-instrumentation/httpclient/arex-httpclient-common/src/main/java/io/arex/inst/httpclient/common/HttpClientExtractor.java b/arex-instrumentation/httpclient/arex-httpclient-common/src/main/java/io/arex/inst/httpclient/common/HttpClientExtractor.java index 976a60a30..c1e59b396 100644 --- a/arex-instrumentation/httpclient/arex-httpclient-common/src/main/java/io/arex/inst/httpclient/common/HttpClientExtractor.java +++ b/arex-instrumentation/httpclient/arex-httpclient-common/src/main/java/io/arex/inst/httpclient/common/HttpClientExtractor.java @@ -2,6 +2,7 @@ import io.arex.agent.bootstrap.model.MockResult; import io.arex.agent.bootstrap.model.Mocker; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.serializer.Serializer; import io.arex.inst.runtime.util.IgnoreUtils; import io.arex.inst.runtime.util.MockUtils; @@ -69,9 +70,9 @@ private Mocker makeMocker() { Map attributes = new HashMap<>(); mocker.getTargetRequest().setAttributes(attributes); - attributes.put("HttpMethod", httpMethod); - attributes.put("QueryString", adapter.getUri().getQuery()); - attributes.put("ContentType", adapter.getRequestContentType()); + attributes.put(ArexConstants.HTTP_METHOD, httpMethod); + attributes.put(ArexConstants.HTTP_QUERY_STRING, adapter.getUri().getQuery()); + attributes.put(ArexConstants.HTTP_CONTENT_TYPE, adapter.getRequestContentType()); mocker.getTargetRequest().setBody(this.encodeRequest(httpMethod)); return mocker; diff --git a/arex-instrumentation/httpclient/arex-httpclient-resttemplate/src/main/java/io/arex/inst/httpclient/resttemplate/RestTemplateExtractor.java b/arex-instrumentation/httpclient/arex-httpclient-resttemplate/src/main/java/io/arex/inst/httpclient/resttemplate/RestTemplateExtractor.java index 2aacf10d7..51c07935d 100644 --- a/arex-instrumentation/httpclient/arex-httpclient-resttemplate/src/main/java/io/arex/inst/httpclient/resttemplate/RestTemplateExtractor.java +++ b/arex-instrumentation/httpclient/arex-httpclient-resttemplate/src/main/java/io/arex/inst/httpclient/resttemplate/RestTemplateExtractor.java @@ -7,6 +7,7 @@ import io.arex.agent.bootstrap.model.MockResult; import io.arex.agent.bootstrap.model.Mocker; import io.arex.inst.runtime.log.LogManager; +import io.arex.inst.runtime.model.ArexConstants; import io.arex.inst.runtime.serializer.Serializer; import io.arex.inst.runtime.util.IgnoreUtils; import io.arex.inst.runtime.util.MockUtils; @@ -14,6 +15,7 @@ import java.net.URI; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RequestCallback; @@ -107,8 +109,8 @@ private Mocker makeMocker() { Map attributes = new HashMap<>(2); mocker.getTargetRequest().setAttributes(attributes); - attributes.put("HttpMethod", httpMethod); - attributes.put("QueryString", uri.getQuery()); + attributes.put(ArexConstants.HTTP_METHOD, Objects.isNull(httpMethod) ? null : httpMethod.name()); + attributes.put(ArexConstants.HTTP_QUERY_STRING, uri.getQuery()); mocker.getTargetRequest().setBody(request); return mocker; diff --git a/arex-instrumentation/netty/arex-netty-v3/src/main/java/io/arex/inst/netty/v3/server/RequestTracingHandler.java b/arex-instrumentation/netty/arex-netty-v3/src/main/java/io/arex/inst/netty/v3/server/RequestTracingHandler.java index f66255fc5..299a72790 100644 --- a/arex-instrumentation/netty/arex-netty-v3/src/main/java/io/arex/inst/netty/v3/server/RequestTracingHandler.java +++ b/arex-instrumentation/netty/arex-netty-v3/src/main/java/io/arex/inst/netty/v3/server/RequestTracingHandler.java @@ -36,6 +36,8 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) throw String excludeMockTemplate = NettyHelper.getHeader(request, ArexConstants.HEADER_EXCLUDE_MOCK); CaseEventDispatcher.onEvent(CaseEvent.ofCreateEvent(EventSource.of(caseId, excludeMockTemplate))); ContextManager.currentContext().setAttachment(ArexConstants.FORCE_RECORD, NettyHelper.getHeader(request, ArexConstants.FORCE_RECORD)); + ContextManager.currentContext().setAttachment(ArexConstants.SCHEDULE_REPLAY, NettyHelper.getHeader(request, ArexConstants.SCHEDULE_REPLAY)); + ContextManager.currentContext().setAttachment(ArexConstants.REPLAY_END_FLAG, NettyHelper.getHeader(request, ArexConstants.REPLAY_END_FLAG)); if (ContextManager.needRecordOrReplay()) { Mocker mocker = MockUtils.createNettyProvider(request.getUri()); Mocker.Target target = mocker.getTargetRequest(); diff --git a/arex-instrumentation/netty/arex-netty-v4/src/main/java/io/arex/inst/netty/v4/server/RequestTracingHandler.java b/arex-instrumentation/netty/arex-netty-v4/src/main/java/io/arex/inst/netty/v4/server/RequestTracingHandler.java index dabf2b4b7..412ca32c3 100644 --- a/arex-instrumentation/netty/arex-netty-v4/src/main/java/io/arex/inst/netty/v4/server/RequestTracingHandler.java +++ b/arex-instrumentation/netty/arex-netty-v4/src/main/java/io/arex/inst/netty/v4/server/RequestTracingHandler.java @@ -38,6 +38,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception String excludeMockTemplate = request.headers().get(ArexConstants.HEADER_EXCLUDE_MOCK); CaseEventDispatcher.onEvent(CaseEvent.ofCreateEvent(EventSource.of(caseId, excludeMockTemplate))); ContextManager.currentContext().setAttachment(ArexConstants.FORCE_RECORD, request.headers().get(ArexConstants.FORCE_RECORD)); + ContextManager.currentContext().setAttachment(ArexConstants.SCHEDULE_REPLAY, request.headers().get(ArexConstants.SCHEDULE_REPLAY)); + ContextManager.currentContext().setAttachment(ArexConstants.REPLAY_END_FLAG, request.headers().get(ArexConstants.REPLAY_END_FLAG)); if (ContextManager.needRecordOrReplay()) { Mocker mocker = MockUtils.createNettyProvider(request.getUri()); Mocker.Target target = mocker.getTargetRequest(); diff --git a/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletAdviceHelper.java b/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletAdviceHelper.java index 81e3b7742..5bbdb9627 100644 --- a/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletAdviceHelper.java +++ b/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletAdviceHelper.java @@ -287,5 +287,6 @@ private static String getRedirectRecordId(ServletAdapter void addAttachmentsToContext(ServletAdapter adapter, TRequest request) { ContextManager.setAttachment(ArexConstants.FORCE_RECORD, adapter.getRequestHeader(request, ArexConstants.FORCE_RECORD, ArexConstants.HEADER_X_PREFIX)); ContextManager.setAttachment(ArexConstants.SCHEDULE_REPLAY, adapter.getRequestHeader(request, ArexConstants.SCHEDULE_REPLAY)); + ContextManager.setAttachment(ArexConstants.REPLAY_END_FLAG, adapter.getRequestHeader(request, ArexConstants.REPLAY_END_FLAG)); } } diff --git a/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletUtil.java b/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletUtil.java index c2ed4505b..7ff6f92ba 100644 --- a/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletUtil.java +++ b/arex-instrumentation/servlet/arex-httpservlet/src/main/java/io/arex/inst/httpservlet/ServletUtil.java @@ -4,7 +4,6 @@ import io.arex.agent.bootstrap.util.CollectionUtil; import io.arex.agent.bootstrap.util.StringUtil; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -20,22 +19,7 @@ private ServletUtil() { } public static String appendUri(String uri, String name, String value) { - try { - URI oldUri = URI.create(uri); - StringBuilder builder = new StringBuilder(); - String newQuery = oldUri.getQuery(); - if (oldUri.getQuery() == null) { - builder.append(name).append("=").append(value); - } else { - builder.append(newQuery).append("&").append(name).append("=").append(value); - } - - URI newUri = new URI(oldUri.getScheme(), oldUri.getAuthority(), - oldUri.getPath(), builder.toString(), oldUri.getFragment()); - return newUri.toString(); - } catch (URISyntaxException e) { - return uri; - } + return UriComponentsBuilder.fromUriString(uri).queryParam(name, value).build().toString(); } public static String getRequestPath(String uri) { diff --git a/arex-instrumentation/servlet/arex-httpservlet/src/test/java/io/arex/inst/httpservlet/ServletUtilTest.java b/arex-instrumentation/servlet/arex-httpservlet/src/test/java/io/arex/inst/httpservlet/ServletUtilTest.java index 90cd0b812..50f3022db 100644 --- a/arex-instrumentation/servlet/arex-httpservlet/src/test/java/io/arex/inst/httpservlet/ServletUtilTest.java +++ b/arex-instrumentation/servlet/arex-httpservlet/src/test/java/io/arex/inst/httpservlet/ServletUtilTest.java @@ -22,6 +22,9 @@ void appendUri() { assertEquals("http://arextest.com?email=arex.test.com@gmail.com&name=mark#fragment", ServletUtil.appendUri("http://arextest.com?email=arex.test.com@gmail.com#fragment", "name", "mark")); + + assertEquals("http://arextest.com?Signature=HJeNHsZ7%2BDMj0JsTK3zd3nzBDQE%3D&name=mark", + ServletUtil.appendUri("http://arextest.com?Signature=HJeNHsZ7%2BDMj0JsTK3zd3nzBDQE%3D", "name", "mark")); } @Test diff --git a/arex-third-party/pom.xml b/arex-third-party/pom.xml index a3f06e35e..da8d03acd 100644 --- a/arex-third-party/pom.xml +++ b/arex-third-party/pom.xml @@ -15,7 +15,20 @@ com.github.jsqlparser jsqlparser - 4.9 + 4.5 + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-simple + + + com.github.luben + zstd-jni + 1.5.2-5 @@ -35,7 +48,7 @@ net.sf.jsqlparser - io.arex.net.sf.jsqlparser + io.arex.shaded.net.sf.jsqlparser diff --git a/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/CompressUtil.java b/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/CompressUtil.java new file mode 100644 index 000000000..b6b73a358 --- /dev/null +++ b/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/CompressUtil.java @@ -0,0 +1,83 @@ +package io.arex.agent.thirdparty.util; + +import com.github.luben.zstd.RecyclingBufferPool; +import com.github.luben.zstd.ZstdInputStreamNoFinalizer; +import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Compress/decompress util + * + * + * @date 2021/11/09 + */ +public class CompressUtil { + public static final int BYTES_BUFFER_LENGTH = 1024; + public static final byte[] ZERO_BYTE = new byte[0]; + private static final Logger LOGGER = LoggerFactory.getLogger(CompressUtil.class); + + public static byte[] zstdCompress(String original, Charset charsetName) { + return zstdCompress(original.getBytes(charsetName)); + } + + /** + * zstd compress + * @param original original string + * @return + */ + public static byte[] zstdCompress(byte[] original) { + if (original == null || original.length == 0) { + return ZERO_BYTE; + } + + try (ByteArrayInputStream byteInputStream = new ByteArrayInputStream(original); + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(byteInputStream.available()); + ZstdOutputStreamNoFinalizer zstdOutputStream = new ZstdOutputStreamNoFinalizer(byteOutputStream, + RecyclingBufferPool.INSTANCE)) { + + byte[] buffer = new byte[BYTES_BUFFER_LENGTH]; + for (int length; (length = byteInputStream.read(buffer, 0, BYTES_BUFFER_LENGTH)) != -1; ) { + zstdOutputStream.write(buffer, 0, length); + } + + zstdOutputStream.flush(); + zstdOutputStream.close(); + return byteOutputStream.toByteArray(); + } catch (Throwable e) { + LOGGER.warn("[[title=arex.compress]]", e); + return ZERO_BYTE; + } + } + + public static String zstdDecompress(InputStream inputStream, Charset charsetName) { + try (ZstdInputStreamNoFinalizer zstdInputStream = new ZstdInputStreamNoFinalizer(inputStream, + RecyclingBufferPool.INSTANCE); + ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(inputStream.available())) { + + byte[] buffer = new byte[BYTES_BUFFER_LENGTH]; + for (int length; (length = zstdInputStream.read(buffer, 0, BYTES_BUFFER_LENGTH)) != -1; ) { + byteOutputStream.write(buffer, 0, length); + } + + return byteOutputStream.toString(charsetName.name()); + } catch (Throwable e) { + LOGGER.warn("[[title=arex.decompress]]", e); + return null; + } + } + + public static String zstdDecompress(byte[] bytes, Charset charsetName) { + return zstdDecompress(new ByteArrayInputStream(bytes), charsetName); + } + + public static String zstdDecompress(byte[] bytes) { + return zstdDecompress(new ByteArrayInputStream(bytes), StandardCharsets.UTF_8); + } +} diff --git a/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/sqlparser/JSqlParserUtil.java b/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/sqlparser/JSqlParserUtil.java index a41d2543e..ceba975b0 100644 --- a/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/sqlparser/JSqlParserUtil.java +++ b/arex-third-party/src/main/java/io/arex/agent/thirdparty/util/sqlparser/JSqlParserUtil.java @@ -17,20 +17,23 @@ public class JSqlParserUtil { * @param sql sql * @return table schema info */ - public static TableSchema parse(String sql) throws Exception { - sql = PATTERN.matcher(sql).replaceAll(" "); - - Statement statement = CCJSqlParserUtil.parse(sql); - - List tableNameList = new TablesNamesFinder().getTableList(statement); - // sort table name - if (tableNameList != null && tableNameList.size() > 1) { - Collections.sort(tableNameList); - } - + public static TableSchema parse(String sql) { TableSchema tableSchema = new TableSchema(); - tableSchema.setAction(getAction(statement)); - tableSchema.setTableNames(tableNameList); + try { + sql = PATTERN.matcher(sql).replaceAll(" "); + + Statement statement = CCJSqlParserUtil.parse(sql); + tableSchema.setAction(getAction(statement)); + + List tableNameList = new TablesNamesFinder().getTableList(statement); + // sort table name + if (tableNameList != null && !tableNameList.isEmpty()) { + Collections.sort(tableNameList); + } + tableSchema.setTableNames(tableNameList); + } catch (Throwable e) { + // ignore error + } return tableSchema; } diff --git a/pom.xml b/pom.xml index af6b2a3b8..cbd86c91e 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,7 @@ arex-integration-tests arex-instrumentation-api arex-third-party + arex-compare @@ -81,6 +82,7 @@ **/constants/**, **/thirdparty/**, **/integrationtest/**, + **/compare/**, **/*Instrumentation.java, **/JJWTGenerator.java @@ -102,6 +104,7 @@ **/wrapper/**, **/thirdparty/**, **/integrationtest/**, + **/compare/**, **/RedisCommandBuilderImpl.java, **/HttpResponseWrapper.java, **/RFutureWrapper.java @@ -140,6 +143,11 @@ arex-third-party ${project.version} + + ${project.groupId} + arex-compare + ${project.version} + org.slf4j slf4j-api