model.jv

// Warning Flags can either be "S" or "O" (see page 7)
constraint WarningFlagList oftype AllowlistConstraint {
	allowlist: [
		"S",
		"O"
	];
}
valuetype WarningFlag oftype text {
	constraints: [
		WarningFlagList
	];
}

// This is an optical property database of refractive indices and dielectric constants (see page 2)
constraint PropertyTypeList oftype AllowlistConstraint {
	allowlist: [
		"Refractive Index",
		"Dielectric Constant"
	];
}
valuetype PropertyType oftype text {
	constraints:
	[
		PropertyTypeList
	];
}

// DateFormat constrained to either YYYY-MM-DD or DD/MM/YYYY
// Date in shared is only YYYY-MM-DD
constraint DateFormatRegex oftype RegexConstraint {
	regex: /\b(?:\d{4}-\d{2}-\d{2}|(?:\d{1,2}\/){2}\d{4})\b/;
}
valuetype DateFormat oftype text {
	constraints:
	[
		DateFormatRegex
	];
}

use {
	BracesRemover
} from "../../shared/composite-blocktypes.jv";

pipeline RefractiveIndicesAndDielectricConstantsPipeline {

	RefractiveIndicesAndDielectricConstantsExtractor
		-> RefractiveIndicesAndDielectricConstantsTextFileInterpreter
		-> RefractiveIndicesAndDielectricConstantsCSVInterpreter
		-> RefractiveIndicesAndDielectricConstantsTableInterpreter
		-> BracesRemover
		-> RefractiveIndicesAndDielectricConstantsDatabaseLoader;


	block RefractiveIndicesAndDielectricConstantsExtractor oftype HttpExtractor {
		url: "https://figshare.com/ndownloader/files/33982973";
	}

	block RefractiveIndicesAndDielectricConstantsTextFileInterpreter oftype TextFileInterpreter { }

	block RefractiveIndicesAndDielectricConstantsCSVInterpreter oftype CSVInterpreter {
		delimiter: ",";
		enclosing: '"';
		enclosingEscape: '"';
	}

	block BracesRemover oftype BracesRemover {
		columnWithBraces: 'values';
	}

	block RefractiveIndicesAndDielectricConstantsTableInterpreter oftype TableInterpreter {
		header: true;
		columns: [
			"DOI" oftype DOI,
			"Date" oftype DateFormat,
			"Journal" oftype text,
			"Title" oftype text,
			"compound" oftype text,
			"extracted_error" oftype text,
			"extracted_value" oftype text,
			"measurement_wavelength" oftype text,
			"normalised_name" oftype text,
			"raw_value" oftype text,
			"specifier" oftype text,
			"values" oftype text,
			"warning" oftype WarningFlag,
			"dielectric_loss" oftype text,
			"measurement_frequency" oftype text,
			"doi_variation" oftype text,
			"original_sentence" oftype text,
			"property_type" oftype PropertyType,
		];
	}

	block RefractiveIndicesAndDielectricConstantsDatabaseLoader oftype SQLiteLoader {
		table: "RefractiveIndicesAndDielectricConstants";
		file: "./RefractiveIndicesAndDielectricConstants.sqlite";
	}
}