diff --git a/commons/src/main/java/com/powsybl/commons/xml/XmlUtil.java b/commons/src/main/java/com/powsybl/commons/xml/XmlUtil.java index fcbda6f04d3..51ce9289aa5 100644 --- a/commons/src/main/java/com/powsybl/commons/xml/XmlUtil.java +++ b/commons/src/main/java/com/powsybl/commons/xml/XmlUtil.java @@ -14,6 +14,7 @@ import javanet.staxutils.IndentingXMLStreamWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilderFactory; @@ -24,6 +25,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.stream.XMLStreamWriter; +import javax.xml.validation.SchemaFactory; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -180,6 +182,21 @@ public static XMLStreamWriter initializeWriter(boolean indent, String indentStri return initializeWriter(indent, indentString, xmlWriter); } + public static SchemaFactory newSchemaFactory() { + SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + try { + factory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + } catch (SAXException e) { + LOGGER.info("- Property unsupported by SchemaFactory implementation: {}", XMLConstants.ACCESS_EXTERNAL_SCHEMA); + } + try { + factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + } catch (SAXException e) { + LOGGER.info("- Property unsupported by SchemaFactory implementation: {}", XMLConstants.ACCESS_EXTERNAL_DTD); + } + return factory; + } + private static XMLStreamWriter initializeWriter(boolean indent, String indentString, XMLStreamWriter initialXmlWriter) throws XMLStreamException { return initializeWriter(indent, indentString, initialXmlWriter, StandardCharsets.UTF_8); } diff --git a/commons/src/test/java/com/powsybl/commons/xml/XmlUtilTest.java b/commons/src/test/java/com/powsybl/commons/xml/XmlUtilTest.java index 38f2127c834..189503ddfcc 100644 --- a/commons/src/test/java/com/powsybl/commons/xml/XmlUtilTest.java +++ b/commons/src/test/java/com/powsybl/commons/xml/XmlUtilTest.java @@ -10,8 +10,12 @@ import com.google.common.collect.ImmutableMap; import com.powsybl.commons.PowsyblException; import org.junit.jupiter.api.Test; +import org.xml.sax.SAXNotRecognizedException; +import org.xml.sax.SAXNotSupportedException; +import javax.xml.XMLConstants; import javax.xml.stream.*; +import javax.xml.validation.SchemaFactory; import java.io.ByteArrayOutputStream; import java.io.StringReader; import java.nio.charset.StandardCharsets; @@ -22,8 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import static com.powsybl.commons.xml.XmlUtil.getXMLInputFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * @author Geoffroy Jamgotchian {@literal } @@ -188,4 +191,19 @@ void initializeWriter() throws XMLStreamException { writer.close(); assertEquals("", baos.toString()); } + + @Test + void testSchemaFactory() { + SchemaFactory factory = XmlUtil.newSchemaFactory(); + assertNotNull(factory); + try { + Object value1 = factory.getProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA); + assertEquals("", value1); + + Object value2 = factory.getProperty(XMLConstants.ACCESS_EXTERNAL_DTD); + assertEquals("", value2); + } catch (SAXNotSupportedException | SAXNotRecognizedException ignored) { + // ignored + } + } } diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java index 67e3a8e456a..979bdc2c802 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java @@ -40,7 +40,9 @@ import org.xml.sax.SAXException; import javax.xml.XMLConstants; +import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.Source; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; @@ -53,13 +55,18 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.ForkJoinPool; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import static com.powsybl.commons.xml.XmlUtil.getXMLInputFactory; +import static com.powsybl.commons.xml.XmlUtil.newSchemaFactory; import static com.powsybl.iidm.serde.AbstractTreeDataImporter.SUFFIX_MAPPING; import static com.powsybl.iidm.serde.IidmSerDeConstants.IIDM_PREFIX; import static com.powsybl.iidm.serde.IidmSerDeConstants.INDENT; @@ -85,7 +92,15 @@ public final class NetworkSerDe { static final byte[] BIIDM_MAGIC_NUMBER = {0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x49, 0x49, 0x44, 0x4d}; private static final Supplier DEFAULT_SCHEMA_SUPPLIER = Suppliers.memoize(() -> NetworkSerDe.createSchema(DefaultExtensionsSupplier.getInstance())); + private static final Supplier> DEFAULT_SCHEMAS_SUPPLIER = Suppliers.memoize(ConcurrentHashMap::new); + private static final int MAX_NAMESPACE_PREFIX_NUM = 100; + private static final String XSD_RESOURCE_DIR = "/xsd/"; + private static final Set ALLOWED_IIDM_XSDS = Stream.of(IidmVersion.values()) + .flatMap(v -> v.supportEquipmentValidationLevel() + ? Stream.of(v.getXsd(), v.getXsd(false)) + : Stream.of(v.getXsd())) + .collect(Collectors.toUnmodifiableSet()); private NetworkSerDe() { } @@ -94,6 +109,10 @@ public static void validate(InputStream is) { validate(is, DefaultExtensionsSupplier.getInstance()); } + public static void validate(InputStream is, IidmVersion version) { + validate(is, version, DefaultExtensionsSupplier.getInstance()); + } + public static void validate(InputStream is, ExtensionsSupplier extensionsSupplier) { Objects.requireNonNull(extensionsSupplier); Schema schema; @@ -125,19 +144,17 @@ private static Schema createSchema(ExtensionsSupplier extensionsSupplier) { for (ExtensionSerDe e : extensionsSupplier.get().getProviders()) { e.getXsdAsStreamList().forEach(xsd -> additionalSchemas.add(new StreamSource(xsd))); } - SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + SchemaFactory factory = newSchemaFactory(); try { - factory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); int length = IidmVersion.values().length + (int) Arrays.stream(IidmVersion.values()) .filter(IidmVersion::supportEquipmentValidationLevel).count(); Source[] sources = new Source[additionalSchemas.size() + length]; int i = 0; int j = 0; for (IidmVersion version : IidmVersion.values()) { - sources[i] = new StreamSource(NetworkSerDe.class.getResourceAsStream("/xsd/" + version.getXsd())); + sources[i] = new StreamSource(NetworkSerDe.class.getResourceAsStream(XSD_RESOURCE_DIR + version.getXsd())); if (version.supportEquipmentValidationLevel()) { - sources[j + IidmVersion.values().length] = new StreamSource(NetworkSerDe.class.getResourceAsStream("/xsd/" + version.getXsd(false))); + sources[j + IidmVersion.values().length] = new StreamSource(NetworkSerDe.class.getResourceAsStream(XSD_RESOURCE_DIR + version.getXsd(false))); j++; } i++; @@ -151,6 +168,198 @@ private static Schema createSchema(ExtensionsSupplier extensionsSupplier) { } } + public static void validate(InputStream is, IidmVersion version, ExtensionsSupplier extensionsSupplier) { + Objects.requireNonNull(is); + Objects.requireNonNull(version); + Objects.requireNonNull(extensionsSupplier); + + // check version namespace + byte[] xmlBytes; + try { + xmlBytes = is.readAllBytes(); + checkNamespace(xmlBytes, version); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + // XSD validation + Schema schema; + if (extensionsSupplier == DefaultExtensionsSupplier.getInstance()) { + schema = DEFAULT_SCHEMAS_SUPPLIER.get().computeIfAbsent(version, v -> createSchema(DefaultExtensionsSupplier.getInstance(), v)); + } else { + schema = createSchema(extensionsSupplier, version); + } + try { + schema.newValidator().validate(new StreamSource(new ByteArrayInputStream(xmlBytes))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (SAXException e) { + throw new UncheckedSaxException(e); + } + } + + private static Schema createSchema(ExtensionsSupplier extensionsSupplier, IidmVersion version) { + Objects.requireNonNull(extensionsSupplier); + Objects.requireNonNull(version); + + SchemaFactory factory = newSchemaFactory(); + try { + List sources = new ArrayList<>(); + // iidm: source + sources.add(new StreamSource(NetworkSerDe.class.getResourceAsStream(XSD_RESOURCE_DIR + version.getXsd()))); + // equipment: source + if (version.supportEquipmentValidationLevel()) { + sources.add(new StreamSource(NetworkSerDe.class.getResourceAsStream(XSD_RESOURCE_DIR + version.getXsd(false)))); + } + // extension: sources + sources.addAll(getExtensionSources(extensionsSupplier, version)); + + return factory.newSchema(sources.toArray(Source[]::new)); + } catch (SAXException e) { + throw new UncheckedSaxException(e); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Build the list of XSD required to validate extensions for a given IIDM version. + * + *

Some extension XSDs import an IIDM schema through {@code xs:import/@schemaLocation}

+ * This method parses each supported extension XSD, extracts the schema locations, + * and adds the corresponding IIDM XSD resources. + *

Only extensions supported by the provided IIDM version are considered.

+ * + * @param extensionsSupplier extension provider used to discover available extension + * @param version IIDM version used to filter compatible extensions + * @return list of additional schema sources required by extension + */ + private static List getExtensionSources(ExtensionsSupplier extensionsSupplier, IidmVersion version) throws IOException { + List sources = new ArrayList<>(); + for (ExtensionSerDe extension : getSupportedExtensionSerDeByIIdmVersion(extensionsSupplier, version)) { + InputStream in = extension.getXsdAsStream(); + byte[] extensionXsd = in.readAllBytes(); + //required iidm xsd in extension's xsd: source + extractSchemaLocations(extensionXsd) + .forEach(schemaLocation -> sources.add(new StreamSource(NetworkSerDe.class.getResourceAsStream(XSD_RESOURCE_DIR + schemaLocation)))); + // extension xsd: source + sources.add(new StreamSource(new ByteArrayInputStream(extensionXsd))); + } + return sources; + } + + private static List> getSupportedExtensionSerDeByIIdmVersion(ExtensionsSupplier extensionsSupplier, IidmVersion version) { + List> extensions = new ArrayList<>(); + for (ExtensionSerDe extensionSerDe : extensionsSupplier.get().getProviders()) { + if (extensionSerDe instanceof AbstractVersionableNetworkExtensionSerDe versionable) { + if (versionable.versionExists(version)) { + extensions.add(extensionSerDe); + } + } else { + // no versionable extensions + extensions.add(extensionSerDe); + } + } + return extensions; + } + + private static void checkNamespace(byte[] xmlBytes, IidmVersion validationVersion) { + String actualNs = readRootNamespace(xmlBytes); + boolean matches = actualNs.equals(validationVersion.getNamespaceURI()) + || validationVersion.supportEquipmentValidationLevel() && actualNs.equals(validationVersion.getNamespaceURI(false)); + if (!matches) { + throw new PowsyblException("Namespace mismatch: expected validation version " + validationVersion.toString(".") + ", found namespace " + actualNs); + } + } + + /** + * Extract {@code xs:import/@schemaLocation} from XSD document + * + *

XSD document snippet:

+ * 
{@code
+     * ...
+     *     targetNamespace="http://www.powsybl.org/schema/iidm/ext/extension-name/1_0"
+     *     xmlns:iidm="http://www.powsybl.org/schema/iidm/1_10">
+     *     
+     * 
+     * }
+ * + * @param xsdBytes XSD content as bytes + * @return schema locations found in {@code xs:import} + */ + private static List extractSchemaLocations(byte[] xsdBytes) { + try { + return proceedExtractSchemaLocations(xsdBytes); + } catch (XMLStreamException e) { + throw new UncheckedXmlStreamException(e); + } + } + + private static List proceedExtractSchemaLocations(byte[] xsdBytes) throws XMLStreamException { + List locations = new ArrayList<>(); + XMLStreamReader reader = null; + try (ByteArrayInputStream in = new ByteArrayInputStream(xsdBytes)) { + reader = getXMLInputFactory().createXMLStreamReader(in); + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT + && XMLConstants.W3C_XML_SCHEMA_NS_URI.equals(reader.getNamespaceURI()) + && "import".equals(reader.getLocalName())) { + String schemaLocation = reader.getAttributeValue(null, "schemaLocation"); + if (schemaLocation != null && !schemaLocation.isBlank() && ALLOWED_IIDM_XSDS.contains(schemaLocation)) { + locations.add(schemaLocation); + } + } + } + return locations; + } catch (XMLStreamException | IOException e) { + throw new PowsyblException("Failed to parse XSD schema", e); + } finally { + if (reader != null) { + reader.close(); + } + } + } + + /** + * Read the namespace declared on {@code } element + * + * @param xmlBytes XML document content as bytes + * @return Namespace URI + */ + private static String readRootNamespace(byte[] xmlBytes) { + try { + return proceedReadRootNamespace(xmlBytes); + } catch (XMLStreamException e) { + throw new UncheckedXmlStreamException(e); + } + } + + private static String proceedReadRootNamespace(byte[] xmlBytes) throws XMLStreamException { + XMLStreamReader reader = null; + try (ByteArrayInputStream in = new ByteArrayInputStream(xmlBytes)) { + reader = getXMLInputFactory().createXMLStreamReader(in); + while (reader.hasNext()) { + if (reader.next() == XMLStreamConstants.START_ELEMENT) { + if (!NETWORK_ROOT_ELEMENT_NAME.equals(reader.getLocalName())) { + throw new PowsyblException("Unexpected root element: " + reader.getLocalName()); + } + String ns = reader.getNamespaceURI(); + if (ns == null || ns.isBlank()) { + throw new PowsyblException("Missing root namespace"); + } + return ns; + } + } + throw new PowsyblException("Missing root namespace"); + } catch (XMLStreamException | IOException e) { + throw new PowsyblException("Failed to read namespace from XML", e); + } finally { + if (reader != null) { + reader.close(); + } + } + } + private static void throwExceptionIfOption(AbstractOptions options, String message) { if (options.isThrowExceptionIfExtensionNotFound()) { throw new PowsyblException(message); diff --git a/iidm/iidm-serde/src/test/java/com/powsybl/iidm/serde/NetworkSerDeTest.java b/iidm/iidm-serde/src/test/java/com/powsybl/iidm/serde/NetworkSerDeTest.java index 61f4a6f8723..5b9c6e787f2 100644 --- a/iidm/iidm-serde/src/test/java/com/powsybl/iidm/serde/NetworkSerDeTest.java +++ b/iidm/iidm-serde/src/test/java/com/powsybl/iidm/serde/NetworkSerDeTest.java @@ -20,6 +20,8 @@ import com.powsybl.commons.test.TestUtil; import com.powsybl.iidm.network.*; import com.powsybl.iidm.network.test.*; +import com.powsybl.iidm.serde.extensions.util.DefaultExtensionsSupplier; +import com.powsybl.iidm.serde.extensions.util.ExtensionsSupplier; import com.powsybl.iidm.serde.extensions.util.NetworkSourceExtension; import com.powsybl.iidm.serde.extensions.util.NetworkSourceExtensionImpl; import org.junit.jupiter.api.Test; @@ -35,6 +37,7 @@ import static com.powsybl.commons.test.ComparisonUtils.assertTxtEquals; import static com.powsybl.iidm.serde.IidmSerDeConstants.CURRENT_IIDM_VERSION; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.*; /** @@ -388,4 +391,103 @@ void testExportWithoutFlatten() { assertEquals(2, merged.getSubnetworks().size()); assertEquals(2, readNetwork.getSubnetworks().size()); } + + @Test + void testValidateByVersionWhenMatchingOldNetwork() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/V1_2/shuntRoundTripRef.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, IidmVersion.V_1_2)); + } + } + + @Test + void testValidateByVersionWhenMatchingRecentNetwork() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/V1_16/shuntRoundTripRef.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)); + } + } + + @Test + void testValidateByVersionWhenMismatchedNetworkVersion() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/V1_16/shuntRoundTripRef.xml")) { + assertNotNull(is); + assertThatThrownBy(() -> NetworkSerDe.validate(is, IidmVersion.V_1_15)) + .isInstanceOf(PowsyblException.class) + .hasMessageContaining("Namespace mismatch: expected validation version 1.15, found namespace http://www.powsybl.org/schema/iidm/1_16"); + } + } + + @Test + void testValidateByVersionWhenInvalidNetwork() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/V1_16/shuntOldTagName.xml")) { + assertNotNull(is); + assertThatThrownBy(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)) + .isInstanceOf(com.powsybl.commons.exceptions.UncheckedSaxException.class) + .hasMessageContaining("Invalid content was found starting with element '{\"http://www.powsybl.org/schema/iidm/1_16\":shunt}'"); + } + } + + @Test + void testValidateByVersionWhenNetworkContainSlackTerminalExtension() throws IOException { + // Given extension: slack_terminal, version 1.5 that require iidm version 1.8 when validate should succeed + try (InputStream is = getClass().getResourceAsStream("/slackTerminal.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)); + } + } + + @Test + void testValidateByVersionWhenNetworkContainTerminalMockExtension() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/V1_16/eurostag-tutorial-example1-with-terminalMock-ext.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)); + } + } + + @Test + void testValidateByVersionWhenInValidExtension() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/branchStatusWrongEnum.xml")) { + assertNotNull(is); + assertThatThrownBy(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)) + .isInstanceOf(com.powsybl.commons.exceptions.UncheckedSaxException.class) + .hasMessageContaining("Value 'TEST' is not facet-valid with respect to enumeration " + + "'[IN_OPERATION, PLANNED_OUTAGE, FORCED_OUTAGE]'. It must be a value from the enumeration."); + } + } + + @Test + void testValidateWithCustomExtensionSupplier() throws IOException { + ExtensionsSupplier customExtensionsSupplier = () -> DefaultExtensionsSupplier.getInstance().get(); + assertNotSame(DefaultExtensionsSupplier.getInstance(), customExtensionsSupplier); + + try (InputStream is = getClass().getResourceAsStream("/V1_16/shuntRoundTripRef.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16, customExtensionsSupplier)); + } + try (InputStream is = getClass().getResourceAsStream("/V1_16/shuntRoundTripRef.xml")) { + assertNotNull(is); + assertDoesNotThrow(() -> NetworkSerDe.validate(is, customExtensionsSupplier)); + } + } + + @Test + void testValidateByVersionWhenMissingNamespace() throws IOException { + try (InputStream is = getClass().getResourceAsStream("/network-without-namespace.xml")) { + assertNotNull(is); + assertThatThrownBy(() -> NetworkSerDe.validate(is, IidmVersion.V_1_16)) + .isInstanceOf(PowsyblException.class) + .hasMessageContaining("Missing root namespace"); + } + } + + @Test + void testValidateWhenParseMalformedXml() { + String xml = " NetworkSerDe.validate(is, IidmVersion.V_1_16)) + .isInstanceOf(PowsyblException.class) + .hasMessageContaining("Failed to read namespace from XML"); + } } diff --git a/iidm/iidm-serde/src/test/resources/branchStatusWrongEnum.xml b/iidm/iidm-serde/src/test/resources/branchStatusWrongEnum.xml new file mode 100644 index 00000000000..f8cd6ad0096 --- /dev/null +++ b/iidm/iidm-serde/src/test/resources/branchStatusWrongEnum.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + TEST + + \ No newline at end of file diff --git a/iidm/iidm-serde/src/test/resources/network-without-namespace.xml b/iidm/iidm-serde/src/test/resources/network-without-namespace.xml new file mode 100644 index 00000000000..7586fb2e44a --- /dev/null +++ b/iidm/iidm-serde/src/test/resources/network-without-namespace.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file