BUG: User-defined plugins not being serialized (Issue #155)

Author: tsegall

ChangeLog.md

@@ -1,6 +1,9 @@
 
 ## Changes ##
 
+### 17.4.2
+ - BUG: User-defined plugins not being serialized (Issue #155)
+
 ### 17.4.1
  - INT: Bump logback-classic to 1.5.27, google phonenumber to 9.0.23, gradle to 9.3.1
  - INT: Cleaning.

examples/sampleplugin/src/main/java/sampleplugin/SamplePlugin.java

@@ -44,10 +44,11 @@ public static void main(final String[] args) throws IOException, FTAException {
 		// Add our custom Java plugin for detecting colors
 		addPlugin(analysis);
 
+		final TextAnalyzer hydratedEnglish = TextAnalyzer.deserialize(analysis.serialize());
 		for (final String input : inputsEnglish)
-			analysis.train(input);
+			hydratedEnglish.train(input);
 
-		result = analysis.getResult();
+		result = hydratedEnglish.getResult();
 
 		System.err.printf("Result: %s, Semantic Type: %s, Regular Expression: %s, Max: %s, Min: %s.%n", result.getType(), result.getSemanticType(), result.getRegExp(), result.getMaxValue(), result.getMinValue());
 
@@ -66,10 +67,12 @@ public static void main(final String[] args) throws IOException, FTAException {
 		// Add our custom Java plugin for detecting colors
 		addPlugin(analysisFrench);
 
+		final TextAnalyzer hydratedFrench = TextAnalyzer.deserialize(analysisFrench.serialize());
+
 		for (final String input : inputsFrench)
-			analysisFrench.train(input);
+			hydratedFrench.train(input);
 
-		result = analysisFrench.getResult();
+		result = hydratedFrench.getResult();
 
 		System.err.printf("Result: %s, Semantic Type: %s, Regular Expression: %s, Max: %s, Min: %s.%n", result.getType(), result.getSemanticType(), result.getRegExp(), result.getMaxValue(), result.getMinValue());
 
@@ -91,21 +94,26 @@ public static void main(final String[] args) throws IOException, FTAException {
 		analyzerAugmented.setLocale(Locale.forLanguageTag("en-IN"));
 		addPlugins(analyzerAugmented);
 
+		TextAnalyzer hydratedAugmented = TextAnalyzer.deserialize(analyzerAugmented.serialize());
+
 		for (final String input : inputsRE)
-			analyzerAugmented.train(input);
+			hydratedAugmented.train(input);
 
-		result = analyzerAugmented.getResult();
+		result = hydratedAugmented.getResult();
 
 		System.err.printf("Result: %s, Semantic Type: %s, Regular Expression: %s, Max: %s, Min: %s.%n", result.getType(), result.getSemanticType(), result.getRegExp(), result.getMaxValue(), result.getMinValue());
 
 		// Load our new plugins from a file and test the new List-based Semantic Type
 		analyzerAugmented = new TextAnalyzer("Element");
 		analyzerAugmented.setLocale(Locale.US);
 		addPlugins(analyzerAugmented);
+
+		hydratedAugmented = TextAnalyzer.deserialize(analyzerAugmented.serialize());
+
 		for (final String input : inputList)
-			analyzerAugmented.train(input);
+			hydratedAugmented.train(input);
 
-		result = analyzerAugmented.getResult();
+		result = hydratedAugmented.getResult();
 
 		System.err.printf("Result: %s, Semantic Type: %s, Regular Expression: %s, Max: %s, Min: %s.%n", result.getType(), result.getSemanticType(), result.getRegExp(), result.getMaxValue(), result.getMinValue());
 	}

settings.gradle

@@ -40,7 +40,7 @@ includeBuild 'examples/core/speed'
 dependencyResolutionManagement {
 	versionCatalogs {
 		libs {
-			version('fta', '17.4.1')
+			version('fta', '17.4.2')
 			version('jacoco', '0.8.12')
 
 // https://mvnrepository.com/artifact/de.siegmar/fastcsv

types/src/main/java/com/cobber/fta/Content.java

@@ -17,6 +17,8 @@
 
 import java.util.Arrays;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
 /**
  * Used to capture the members for a set of items - defined either inline or via a file or resource.
  */
@@ -43,6 +45,7 @@ public Content(final String[] members) {
 		this.members = members;
 	}
 
+	@JsonIgnore
 	public String getCacheKey() {
 		if (INLINE.equals(type))
 			return type + "---" + Arrays.hashCode(members);

types/src/main/java/com/cobber/fta/PluginDefinition.java

@@ -30,6 +30,7 @@
 import com.cobber.fta.core.FTAType;
 import com.cobber.fta.core.HeaderEntry;
 import com.cobber.fta.core.InternalErrorException;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -111,6 +112,7 @@ public PluginDefinition() {
 	 *
 	 * @return The order for this plugin.
 	 */
+	@JsonIgnore
 	public int getOrder() {
 		switch (precedence) {
 			case BUILTIN:
@@ -276,6 +278,7 @@ public boolean isLocaleSupported(final Locale locale) throws FTAPluginException
 		return getLocaleEntry(locale) != null;
 	}
 
+	@JsonIgnore
 	public String getLocaleDescription() {
 		final StringBuilder ret = new StringBuilder();
 		for (final PluginLocaleEntry validLocale : validLocales)

types/src/main/java/com/cobber/fta/PluginMatchEntry.java

@@ -15,6 +15,8 @@
  */
 package com.cobber.fta;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 /*
  * Used by RegExp plugins.  Each RegExp plugin has a set of PluginMatchEntries which have an ordered list of regular Expressions used to match against the Stream Data.
  * If the data matches one of the regular expressions then the regExpReturned is the answer returned to the user.  For example for a SSN the "regExpsToMatch" is
@@ -27,7 +29,8 @@ public class PluginMatchEntry {
 	/** RegExp plugins: the RegExp to be returned for this Semantic Type. */
 	public String regExpReturned;
 	/** Is the returned Regular Expression a complete representation of the Semantic Type. */
-	public boolean isRegExpComplete;
+	@JsonProperty("isRegExpComplete")
+	public boolean regExpComplete;
 
 	PluginMatchEntry() {
 	}
@@ -43,7 +46,7 @@ public class PluginMatchEntry {
 	PluginMatchEntry(final PluginMatchEntry other) {
 		this.regExpsToMatch = other.regExpsToMatch == null ? null : other.regExpsToMatch.clone();
 		this.regExpReturned = other.regExpReturned;
-		this.isRegExpComplete = other.isRegExpComplete;
+		this.regExpComplete = other.regExpComplete;
 	}
 
 	public String[] getRegExpsToMatch() {
@@ -56,7 +59,7 @@ public String[] getRegExpsToMatch() {
 	}
 
 	public boolean isRegExpComplete() {
-		return isRegExpComplete;
+		return regExpComplete;
 	}
 
 	public String getRegExpReturned() {

types/src/main/java/com/cobber/fta/TextAnalyzer.java

@@ -3775,7 +3775,7 @@ public String serialize() throws FTAPluginException, FTAUnsupportedLocaleExcepti
 		if (facts.getMatchTypeInfo() == null)
 			determineType();
 
-		final TextAnalyzerWrapper wrapper = new TextAnalyzerWrapper(analysisConfig, context, facts.calculateFacts());
+		final TextAnalyzerWrapper wrapper = new TextAnalyzerWrapper(analysisConfig, context, getPlugins().getUserDefinedPlugins(), facts.calculateFacts());
 
 		// We are serializing the analyzer (assume it will not be used again - so persist the samples)
 		if (traceConfig != null) {
@@ -3809,6 +3809,7 @@ public static TextAnalyzer deserialize(final String serialized) throws FTAMergeE
 
 			ret.facts = wrapper.facts;
 			ret.facts.setConfig(wrapper.analysisConfig);
+			ret.getPlugins().registerPluginListWithPrecedence(wrapper.userDefinedPlugins, wrapper.analysisConfig);
 			ret.initializeTrace();
 			ret.initialize();
 			ret.facts.hydrate();

types/src/main/java/com/cobber/fta/TextAnalyzerWrapper.java

@@ -15,20 +15,24 @@
  */
 package com.cobber.fta;
 
+import java.util.List;
+
 /**
  * Simple class used to aid serialization/de-serialization of a TextAnalyzer.
  */
 class TextAnalyzerWrapper {
 	public AnalysisConfig analysisConfig;
 	public AnalyzerContext analyzerContext;
 	public Facts facts;
+	public List<PluginDefinition> userDefinedPlugins;
 
 	TextAnalyzerWrapper() {
 	}
 
-	TextAnalyzerWrapper(final AnalysisConfig analysisConfig, final AnalyzerContext analyzerContext, final Facts facts) {
+	TextAnalyzerWrapper(final AnalysisConfig analysisConfig, final AnalyzerContext analyzerContext, final List<PluginDefinition> userDefinedPlugins, final Facts facts) {
 		this.analysisConfig = analysisConfig;
 		this.analyzerContext = analyzerContext;
 		this.facts = facts;
+		this.userDefinedPlugins = userDefinedPlugins;
 	}
 }

types/src/test/java/com/cobber/fta/TestIssues.java

@@ -425,11 +425,99 @@ public void checkJSON() throws IOException, FTAException {
 	final String[] inputsRE = {
             "2345:AQ", "5993:FG", "3898:WW", "5543:NH", "1992:WW", "4002:CS", "5982:KG", "1090:DD", "3030:XX", "1088:TR",
             "2547:DE", "6587:DS", "3215:QQ", "7745:VD", "4562:DD", "4582:SS", "2257:WE", "3578:HT", "4568:FB", "1587:SW",
-            "4573:LF", "3574:SS", "8122:GK", "4523:EW", "7128:RT", "2548:RF", "6873:HH", "4837:NR", "2358:EE", "3731:HY"
+            "4573:LF", "3574:SS", "8122:GK", "4523:EW", "7128:RT", "2548:RF", "6873:HH", "4837:NR", "2358:EE", "3731:HY",
+            "0010:AA", "0011:DB", "0012:FT", "0013:KG", "0014:EM", "0015:RP", "0016:TA", "0017:AP", "0018:AA", "0019:AA",
+            "0020:QA", "0021:UT", "0022:AA", "0023:AA", "0024:AQ", "0025:PA", "0026:AA", "0027:AA", "0028:AQ", "0029:AG",
+            "0030:OA", "0031:AA", "0032:AA", "0033:AA", "0034:AA", "0035:NA", "0036:AI", "0037:AI", "0038:AA", "0039:FA",
+            "0040:AH", "0041:AL", "0042:AL", "0043:EA", "0044:AA", "0045:WA", "0046:AA", "0047:QA", "0048:AS", "0049:XA",
+            "0050:AS", "0051:ZA", "0052:AA", "0053:BS", "0054:JA", "0055:MA", "0056:AA", "0057:AV", "0058:AC", "0059:AA",
+            "0060:AA", "0061:AC", "0062:AA", "0063:AD", "0064:AF", "0065:AA", "0066:AZ", "0067:AA", "0068:AK", "0069:AI",
+            "0070:AA", "0071:AA", "0072:AA", "0073:AA", "0074:AF", "0075:AG", "0076:AY", "0077:AI", "0078:AU", "0079:AA"
     };
 
+	final String[] primaryColors = {
+		"Red", "Green", "Green", "Red", "Red", "Blue", "Green", "Green", "Green", "Blue",
+		"Green", "Blue", "Blue", "Red", "Red", "Blue", "Green", "Blue", "Blue", "Red",
+		"Blue", "Green", "Blue"
+	};
+
+	@Test(groups = { TestGroups.ALL, TestGroups.RANDOM })
+	public void issue155_serialize_pre() throws FTAPluginException, FTAUnsupportedLocaleException, FTAMergeException {
+		// Load our new plugins from a file and test the new Regular Expression Semantic Type
+		TextAnalyzer analysis = new TextAnalyzer("ID");
+		analysis.setLocale(Locale.forLanguageTag("en-US"));
+
+		// Register our sample list and regex plugins from a JSON definition file (before the built-in plugins have been registered)
+		try (BufferedReader reader = new BufferedReader(new InputStreamReader(TextAnalyzer.class.getResourceAsStream("/custom_id.json"), StandardCharsets.UTF_8))) {
+				analysis.getPlugins().registerPlugins(reader, analysis.getConfig(), true);
+		} catch (FTAPluginException e) {
+			System.err.println("ERROR: Failed to register plugin: " + (e.getCause() != null ? e.getCause().getMessage() : e.getMessage()));
+		} catch (IOException e) {
+			System.err.println("ERROR: Failed to register plugin: " + e.getMessage());
+		}
+
+		TextAnalyzer hydrated = TextAnalyzer.deserialize(analysis.serialize());
+
+		for (final String input : inputsRE)
+			hydrated.train(input);
+
+		TextAnalysisResult result = hydrated.getResult();
+
+		assertEquals(result.getSemanticType(), "CUSTOM.DIGIT_ALPHA_ID");
+	}
+
+	@Test(groups = { TestGroups.ALL, TestGroups.RANDOM })
+	public void issue155_serialize_color_pre() throws FTAPluginException, FTAUnsupportedLocaleException, FTAMergeException {
+		// Load our new plugins from a file and test the new Regular Expression Semantic Type
+		TextAnalyzer analysis = new TextAnalyzer("Color");
+		analysis.setLocale(Locale.forLanguageTag("en-US"));
+
+		// Register our sample list and regex plugins from a JSON definition file (before the built-in plugins have been registered)
+		try (BufferedReader reader = new BufferedReader(new InputStreamReader(TextAnalyzer.class.getResourceAsStream("/custom_id.json"), StandardCharsets.UTF_8))) {
+				analysis.getPlugins().registerPlugins(reader, analysis.getConfig(), true);
+		} catch (FTAPluginException e) {
+			System.err.println("ERROR: Failed to register plugin: " + (e.getCause() != null ? e.getCause().getMessage() : e.getMessage()));
+		} catch (IOException e) {
+			System.err.println("ERROR: Failed to register plugin: " + e.getMessage());
+		}
+
+		TextAnalyzer hydrated = TextAnalyzer.deserialize(analysis.serialize());
+
+		for (final String input : primaryColors)
+			hydrated.train(input);
+
+		TextAnalysisResult result = hydrated.getResult();
+
+		assertEquals(result.getSemanticType(), "CUSTOM.PRIMARY_COLOR");
+	}
+
+	@Test(groups = { TestGroups.ALL, TestGroups.RANDOM })
+	public void issue155_serialize_color_post() throws FTAPluginException, FTAUnsupportedLocaleException, FTAMergeException {
+		// Load our new plugins from a file and test the new Regular Expression Semantic Type
+		TextAnalyzer analysis = new TextAnalyzer("Color");
+		analysis.setLocale(Locale.forLanguageTag("en-US"));
+
+		// Register our sample list and regex plugins from a JSON definition file (before the built-in plugins have been registered)
+		try (BufferedReader reader = new BufferedReader(new InputStreamReader(TextAnalyzer.class.getResourceAsStream("/custom_id.json"), StandardCharsets.UTF_8))) {
+				analysis.getPlugins().registerPlugins(reader, analysis.getConfig(), false);
+		} catch (FTAPluginException e) {
+			System.err.println("ERROR: Failed to register plugin: " + (e.getCause() != null ? e.getCause().getMessage() : e.getMessage()));
+		} catch (IOException e) {
+			System.err.println("ERROR: Failed to register plugin: " + e.getMessage());
+		}
+
+		TextAnalyzer hydrated = TextAnalyzer.deserialize(analysis.serialize());
+
+		for (final String input : primaryColors)
+			hydrated.train(input);
+
+		TextAnalysisResult result = hydrated.getResult();
+
+		assertEquals(result.getSemanticType(), "COLOR.TEXT_EN");
+	}
+
 	@Test(groups = { TestGroups.ALL, TestGroups.RANDOM })
-	public void issue155() throws FTAPluginException, FTAUnsupportedLocaleException, FTAMergeException {
+	public void issue155_merge() throws FTAPluginException, FTAUnsupportedLocaleException, FTAMergeException {
 		// Load our new plugins from a file and test the new Regular Expression Semantic Type
 		TextAnalyzer shardOne = new TextAnalyzer("ID");
 		shardOne.setLocale(Locale.forLanguageTag("en-US"));
@@ -457,7 +545,7 @@ public void issue155() throws FTAPluginException, FTAUnsupportedLocaleException,
 			shardTwo.train(input);
 		}
 
-		TextAnalyzer merged = TextAnalyzer.merge(shardOne, shardTwo);
+		TextAnalyzer merged = TextAnalyzer.merge(TextAnalyzer.deserialize(shardOne.serialize()), TextAnalyzer.deserialize(shardTwo.serialize()));
 		TextAnalysisResult result = merged.getResult();
 
 		assertEquals(result.getSemanticType(), "CUSTOM.DIGIT_ALPHA_ID");

types/src/test/resources/custom_id.json

@@ -10,5 +10,17 @@
 		} ],
 	"threshold": 95,
 	"baseType": "STRING"
+},
+{
+	"semanticType": "CUSTOM.PRIMARY_COLOR",
+	"description": "Additive primary colors",
+	"pluginType": "list",
+	"validLocales": [ { "localeTag": "*", "headerRegExps": [ { "regExp": ".*(?i)(color|colour).*", "confidence": 90 } ] } ],
+	"content": {
+		"type": "inline",
+		"members": [ "RED", "GREEN", "BLUE" ]
+	},
+	"backout": "[ \\p{IsAlphabetic}]+",
+	"threshold": 99
 }
 ]