Check for better RegExpTypes even if we have already detected a Semantic Type + upgrade libs + remove use of deprecated methods

Author: tsegall

ChangeLog.md

@@ -1,6 +1,13 @@
 
 ## Changes ##
 
+### 17.2.0
+ - INT: Bump logback-classic to 1.5.24, gradle to 9.3.0, phonenumber to 9.0.21
+ - CLI: Add --semanticTypesPre to control whether user-defined types (--semanticType) are registered before built-ins
+ - BUG: findByName on TextAnalyzer was not returning user-defined types
+ - BUG: Preserve precedence on plugins registered on TextAnalyzer template for RecordAnalyzer (Issue #152)
+ - BUG: Check for better RegExpTypes even if we have already detected a Semantic Type (Issue #152)
+ - INT: Remove remaining references to deprecated registerPluginList()
 
 ### 17.1.10
  - INT: Bump logback-classic to 1.5.23, SpotBugs to 6.4.8, commons-text:1.15.0, google phonenumber to 9.0.21,

README.md

@@ -370,6 +370,8 @@ If the pluginType is 'java' then a *clazz* field is required and a new Instance
 If the pluginType is 'list' then the *content* field must be present and a new instance of LogicalTypeFiniteSimpleExternal will be instantiated.
 If the pluginType is 'regex' then an instance of LogicalTypeRegExp will be instantiated.
 
+The JSON schema for plugin definitions is available [here](types/src/main/resources/reference/plugins-schema.json).
+
 In all cases the plugin definition and locale are passed as arguments.
 
 ### All Plugins ###

build.gradle

@@ -4,7 +4,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '9.2.1'
+	gradleVersion = '9.3.0'
 }
 
 tasks.register('examples.clean') {

cli/src/main/java/com/cobber/fta/driver/Driver.java

@@ -103,7 +103,7 @@ else if ("--help".equals(unprocessed[idx])) {
 					error.println(" --noAnalysis - Do not do analysis");
 					error.println(" --noPretty - Do not pretty print analysis");
 					error.println(" --noQuantiles - Do not track quantiles");
-					error.println(" --noSemanticTypes - Do not register any Semantic Types");
+					error.println(" --noSemanticTypes - Do not register any of the built-in Semantic Types");
 					error.println(" --noStatistics - Do not track statistics");
 					error.println(" --pluginDefinition - Output the plugin definitions from the training data set");
 					error.println(" --pluginMode true|false - Set the detect mode when running Plugin validate");
@@ -115,6 +115,7 @@ else if ("--help".equals(unprocessed[idx])) {
 					error.println(" --resolutionMode <DayFirst|MonthFirst|Auto|None> - Auto DayFirst or MonthFirst is determined from Locale");
 					error.println(" --samples - If set then generate samples (see --faker for comprehensive support)");
 					error.println(" --semanticType <JSON representation of Semantic Types> - Can be inline or as a File");
+					error.println(" --semanticTypesPre - If set then user-supplied Semantic Types (--semanticType) are registered ahead of built-ins");
 					error.println(" --signature - Output the Signature for the supplied pluginName");
 					error.println(" --skip <n> - Skip the initial <n> rows of the input");
 					error.println(" --testMerge <n> - exercise merging of analyses, <n> is the number of samples per merge");

cli/src/main/java/com/cobber/fta/driver/DriverOptions.java

@@ -36,6 +36,7 @@ public class DriverOptions {
 	protected int col = -1;
 	protected int debug = -1;
 	protected String semanticTypes;
+	protected boolean semanticTypesPre;
 	// Used to pass in a list of known Semantic Types - e.g. User-stated, or previously identified in some manner
 	protected String knownTypes;
 	protected boolean noAnalysis;
@@ -110,6 +111,7 @@ public DriverOptions(final DriverOptions other) {
 		this.pluginMode = other.pluginMode;
 		this.resolutionMode = other.resolutionMode;
 		this.samples = other.samples;
+		this.semanticTypesPre = other.semanticTypesPre;
 		this.signature = other.signature;
 		this.skip = other.skip;
 		this.threshold = other.threshold;
@@ -164,14 +166,14 @@ public void apply(final TextAnalyzer analyzer) throws IOException {
 				// If the argument starts with a '[' assume it is an inline definition, if not assume it is a file
 				if (this.semanticTypes.charAt(0) == '[')
 					analyzer.getPlugins().registerPlugins(new StringReader(this.semanticTypes),
-							analyzer.getStreamName(), analyzer.getConfig());
+							analyzer.getStreamName(), analyzer.getConfig(), semanticTypesPre);
 				else {
 					if(!Files.isRegularFile(Paths.get(this.semanticTypes))) {
 						System.err.println("ERROR: Failed to read Semantic Types file: " + this.semanticTypes);
 						System.exit(1);
 					}
 					try (FileReader logicalTypes = new FileReader(this.semanticTypes, StandardCharsets.UTF_8)) {
-						analyzer.getPlugins().registerPlugins(logicalTypes, analyzer.getStreamName(), analyzer.getConfig());
+						analyzer.getPlugins().registerPlugins(logicalTypes, analyzer.getStreamName(), analyzer.getConfig(), semanticTypesPre);
 					}
 				}
 			} catch (SecurityException | FTAPluginException e) {
@@ -314,6 +316,8 @@ else if ("--samples".equals(args[idx]))
 				samples = true;
 			else if ("--semanticType".equals(args[idx]))
 				semanticTypes = nextStringArg(args, idx++);
+			else if ("--semanticTypesPre".equals(args[idx]))
+				semanticTypesPre = true;
 			else if ("--signature".equals(args[idx]))
 				signature = true;
 			else if ("--skip".equals(args[idx]))

cli/src/main/java/com/cobber/fta/driver/Processor.java

@@ -42,16 +42,6 @@ public class Processor {
 		this.options = options;
 		this.streamCount = fieldNames.length;
 
-		if (options.pluginName != null && options.pluginMode != null) {
-			final PluginDefinition pluginDefinition = PluginDefinition.findByName(options.pluginName);
-			if (pluginDefinition == null) {
-				logger.printf("ERROR: Failed to locate plugin named '%s', use --help%n", options.pluginName);
-				System.exit(1);
-			}
-
-			logicalType = LogicalTypeFactory.newInstance(pluginDefinition, new AnalysisConfig(options.getLocale()));
-		}
-
 		if (options.col == -1) {
 			final AnalyzerContext context = new AnalyzerContext(null, options.resolutionMode, compositeName, fieldNames);
 			if (options.knownTypes != null)
@@ -64,14 +54,19 @@ public class Processor {
 		}
 
 		analyzers = new TextAnalyzer[streamCount];
-		for (int i = 0; i < fieldNames.length; i++) {
-			if (options.col == -1 || options.col == i) {
-				final AnalyzerContext context = new AnalyzerContext(fieldNames[i] == null ? "" : fieldNames[i].trim(), options.resolutionMode, compositeName, fieldNames);
-				if (options.knownTypes != null)
-					context.withSemanticTypes(options.knownTypes.split(","));
-				analyzers[i] = new TextAnalyzer(context);
-				options.apply(analyzers[i]);
+		final AnalyzerContext context = new AnalyzerContext(fieldNames[options.col] == null ? "" : fieldNames[options.col].trim(), options.resolutionMode, compositeName, fieldNames);
+		if (options.knownTypes != null)
+			context.withSemanticTypes(options.knownTypes.split(","));
+		analyzers[options.col] = new TextAnalyzer(context);
+		options.apply(analyzers[options.col]);
+		if (options.pluginName != null && options.pluginMode != null) {
+			final PluginDefinition pluginDefinition = analyzers[options.col].findByName(options.pluginName);
+			if (pluginDefinition == null) {
+				logger.printf("ERROR: Failed to locate plugin named '%s', use --help%n", options.pluginName);
+				System.exit(1);
 			}
+
+			logicalType = LogicalTypeFactory.newInstance(pluginDefinition, new AnalysisConfig(options.getLocale()));
 		}
 	}
 

examples/contextual/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '9.2.1'
+	gradleVersion = '9.3.0'
 }
 
 repositories {

examples/core/datedemo/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '9.2.1'
+	gradleVersion = '9.3.0'
 }
 
 repositories {

examples/core/dateformat/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '9.2.1'
+	gradleVersion = '9.3.0'
 }
 
 repositories {

examples/core/dateformattrained/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '9.2.1'
+	gradleVersion = '9.3.0'
 }
 
 repositories {