Merge branch 'main' into es-11331-streams-params-restriction

Author: masseyke

.buildkite/pipelines/periodic.template.yml

@@ -246,7 +246,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -684,7 +684,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

build-tools-internal/src/main/resources/forbidden/jdk-signatures.txt

@@ -102,3 +102,29 @@ java.util.Collections#shuffle(java.util.List) @ Use java.util.Collections#shuffl
 
 @defaultMessage Avoid creating FilePermission objects directly, but use FilePermissionUtils instead
 java.io.FilePermission#<init>(java.lang.String,java.lang.String)
+
+@defaultMessage DocumentBuilderFactory should not be used directly. Use XmlUtils#getHardenedDocumentBuilder(java.lang.String[]) instead.
+javax.xml.parsers.DocumentBuilderFactory#newInstance()
+javax.xml.parsers.DocumentBuilderFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+javax.xml.parsers.DocumentBuilderFactory#newDefaultNSInstance()
+javax.xml.parsers.DocumentBuilderFactory#newNSInstance()
+javax.xml.parsers.DocumentBuilderFactory#newNSInstance(java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage TransformerFactory should not be used directly. Use XmlUtils#getHardenedXMLTransformer() instead.
+javax.xml.transform.TransformerFactory#newInstance()
+javax.xml.transform.TransformerFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage SAXParserFactory should not be used directly. Use XmlUtils#getHardenedSaxParser() instead
+javax.xml.parsers.SAXParserFactory#newInstance()
+javax.xml.parsers.SAXParserFactory#newInstance(java.lang.String, java.lang.ClassLoader)
+javax.xml.parsers.SAXParserFactory#newDefaultNSInstance()
+javax.xml.parsers.SAXParserFactory#newNSInstance()
+javax.xml.parsers.SAXParserFactory#newNSInstance(java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage SchemaValidator should not be used directly. Use XmlUtils#getHardenedSchemaValidator() instead
+javax.xml.validation.SchemaFactory#newDefaultInstance()
+javax.xml.validation.SchemaFactory#newInstance(java.lang.String)
+javax.xml.validation.SchemaFactory#newInstance(java.lang.String, java.lang.String, java.lang.ClassLoader)
+
+@defaultMessage Validator should not be used directly. Use XmlUtils#getHardenedValidator() instead
+javax.xml.validation.Schema#newValidator()

docs/changelog/133718.yaml

@@ -0,0 +1,5 @@
+pr: 133718
+summary: Remove upper limit for chunking settings
+area: Machine Learning
+type: enhancement
+issues: []

libs/core/src/main/java/module-info.java

@@ -12,6 +12,7 @@
 module org.elasticsearch.base {
     requires static jsr305;
     requires org.elasticsearch.logging;
+    requires java.xml;
 
     exports org.elasticsearch.core;
     exports org.elasticsearch.jdk;

libs/core/src/main/java/org/elasticsearch/core/XmlUtils.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.xml.sax.SAXParseException;
+
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
+import javax.xml.validation.Validator;
+
+public class XmlUtils {
+
+    private static final Logger LOGGER = LogManager.getLogger(XmlUtils.class);
+
+    /**
+     * Constructs a DocumentBuilder with all the necessary features for it to be secure
+     *
+     * @throws ParserConfigurationException if one of the features can't be set on the DocumentBuilderFactory
+     */
+    public static DocumentBuilder getHardenedBuilder(String[] schemaFiles) throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = getHardenedBuilderFactory();
+        dbf.setNamespaceAware(true);
+        // Ensure that Schema Validation is enabled for the factory
+        dbf.setValidating(true);
+        // This is required, otherwise schema validation causes signature invalidation
+        dbf.setFeature("http://apache.org/xml/features/validation/schema/normalized-value", false);
+        // Make sure that URL schema namespaces are not resolved/downloaded from URLs we do not control
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "file,jar");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "file,jar");
+        // We ship our own xsd files for schema validation since we do not trust anyone else.
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaSource", schemaFiles);
+        DocumentBuilder documentBuilder = dbf.newDocumentBuilder();
+        documentBuilder.setErrorHandler(new ErrorHandler());
+        return documentBuilder;
+    }
+
+    /**
+     * Returns a DocumentBuilderFactory pre-configured to be secure
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a DocumentBuilder")
+    public static DocumentBuilderFactory getHardenedBuilderFactory() throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        // Disallow internal and external entity expansion
+        dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        dbf.setFeature("http://xml.org/sax/features/validation", true);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false);
+        dbf.setIgnoringComments(true);
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        dbf.setFeature("http://apache.org/xml/features/honour-all-schemaLocations", true);
+        // Ensure we do not resolve XIncludes. Defaults to false, but set it explicitly to be future-proof
+        dbf.setXIncludeAware(false);
+        // Ensure we do not expand entity reference nodes
+        dbf.setExpandEntityReferences(false);
+        // Further limit danger from denial of service attacks
+        dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema", true);
+        dbf.setAttribute("http://apache.org/xml/features/validation/schema-full-checking", true);
+        dbf.setAttribute("http://java.sun.com/xml/jaxp/properties/schemaLanguage", XMLConstants.W3C_XML_SCHEMA_NS_URI);
+
+        return dbf;
+    }
+
+    /**
+     * Constructs a Transformer configured to be secure
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a Transformer")
+    public static Transformer getHardenedXMLTransformer() throws TransformerConfigurationException {
+        final TransformerFactory tfactory = TransformerFactory.newInstance();
+        tfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        tfactory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+        tfactory.setAttribute("indent-number", 2);
+        Transformer transformer = tfactory.newTransformer();
+        transformer.setErrorListener(new ErrorListener());
+        return transformer;
+    }
+
+    /**
+     * Returns a SchemaFactory configured to be secure
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a SchemaFactory")
+    public static SchemaFactory getHardenedSchemaFactory() throws SAXNotSupportedException, SAXNotRecognizedException {
+        SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+        schemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        schemaFactory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        return schemaFactory;
+    }
+
+    /**
+     * Constructs a Validator configured to be secure
+     */
+    @SuppressForbidden(reason = "This is the only allowed way to construct a Validator")
+    public static Validator getHardenedValidator(Schema schema) throws SAXNotSupportedException, SAXNotRecognizedException {
+        Validator validator = schema.newValidator();
+        validator.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        validator.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        return validator;
+    }
+
+    @SuppressForbidden(reason = "This is the only allowed way to construct a SAXParser")
+    public static SAXParserFactory getHardenedSaxParserFactory() throws SAXNotSupportedException, SAXNotRecognizedException,
+        ParserConfigurationException {
+        var saxParserFactory = SAXParserFactory.newInstance();
+
+        saxParserFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        saxParserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        saxParserFactory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        saxParserFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        saxParserFactory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+
+        return saxParserFactory;
+    }
+
+    private static class ErrorHandler implements org.xml.sax.ErrorHandler {
+        /**
+         * Enabling schema validation with `setValidating(true)` in our
+         * DocumentBuilderFactory requires that we provide our own
+         * ErrorHandler implementation
+         *
+         * @throws SAXException If the document we attempt to parse is not valid according to the specified schema.
+         */
+        @Override
+        public void warning(SAXParseException e) throws SAXException {
+            LOGGER.debug("XML Parser error ", e);
+            throw e;
+        }
+
+        @Override
+        public void error(SAXParseException e) throws SAXException {
+            LOGGER.debug("XML Parser error ", e);
+            throw e;
+        }
+
+        @Override
+        public void fatalError(SAXParseException e) throws SAXException {
+            LOGGER.debug("XML Parser error ", e);
+            throw e;
+        }
+    }
+
+    private static class ErrorListener implements javax.xml.transform.ErrorListener {
+
+        @Override
+        public void warning(TransformerException e) throws TransformerException {
+            LOGGER.debug("XML transformation error", e);
+            throw e;
+        }
+
+        @Override
+        public void error(TransformerException e) throws TransformerException {
+            LOGGER.debug("XML transformation error", e);
+            throw e;
+        }
+
+        @Override
+        public void fatalError(TransformerException e) throws TransformerException {
+            LOGGER.debug("XML transformation error", e);
+            throw e;
+        }
+    }
+}

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/AbstractExponentialHistogram.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V., and/or licensed to Elasticsearch B.V.
+ * under one or more license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This file is based on a modification of https://github.com/open-telemetry/opentelemetry-java which is licensed under the Apache 2.0 License.
+ */
+
+package org.elasticsearch.exponentialhistogram;
+
+/**
+ * Basic implementation for {@link ExponentialHistogram} with common functionality.
+ */
+public abstract class AbstractExponentialHistogram implements ExponentialHistogram {
+
+    @Override
+    public long valueCount() {
+        return zeroBucket().count() + negativeBuckets().valueCount() + positiveBuckets().valueCount();
+    }
+
+    @Override
+    public int hashCode() {
+        return ExponentialHistogram.hashCode(this);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof ExponentialHistogram) {
+            return ExponentialHistogram.equals(this, (ExponentialHistogram) obj);
+        }
+        return false;
+    }
+}

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java

@@ -23,7 +23,7 @@
 
 import java.util.OptionalLong;
 
-class EmptyExponentialHistogram implements ReleasableExponentialHistogram {
+class EmptyExponentialHistogram extends AbstractExponentialHistogram implements ReleasableExponentialHistogram {
 
     static final EmptyExponentialHistogram INSTANCE = new EmptyExponentialHistogram();
 

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java

@@ -102,6 +102,14 @@ public interface ExponentialHistogram extends Accountable {
      */
     double sum();
 
+    /**
+     * Returns the number of values represented by this histogram.
+     * In other words, this is the sum of the counts of all buckets including the zero bucket.
+     *
+     * @return the value count, guaranteed to be zero for empty histograms
+     */
+    long valueCount();
+
     /**
      * Returns minimum of all values represented by this histogram.
      *
@@ -132,6 +140,60 @@ interface Buckets {
 
     }
 
+    /**
+     * Value-based equality for exponential histograms.
+     * @param a the first histogram (can be null)
+     * @param b the second histogram (can be null)
+     * @return true, if both histograms are equal
+     */
+    static boolean equals(ExponentialHistogram a, ExponentialHistogram b) {
+        if (a == b) return true;
+        if (a == null) return false;
+        if (b == null) return false;
+
+        return a.scale() == b.scale()
+            && a.sum() == b.sum()
+            && equalsIncludingNaN(a.min(), b.min())
+            && a.zeroBucket().equals(b.zeroBucket())
+            && bucketIteratorsEqual(a.negativeBuckets().iterator(), b.negativeBuckets().iterator())
+            && bucketIteratorsEqual(a.positiveBuckets().iterator(), b.positiveBuckets().iterator());
+    }
+
+    private static boolean equalsIncludingNaN(double a, double b) {
+        return (a == b) || (Double.isNaN(a) && Double.isNaN(b));
+    }
+
+    private static boolean bucketIteratorsEqual(BucketIterator a, BucketIterator b) {
+        if (a.scale() != b.scale()) {
+            return false;
+        }
+        while (a.hasNext() && b.hasNext()) {
+            if (a.peekIndex() != b.peekIndex() || a.peekCount() != b.peekCount()) {
+                return false;
+            }
+            a.advance();
+            b.advance();
+        }
+        return a.hasNext() == b.hasNext();
+    }
+
+    /**
+     * Default hash code implementation to be used with {@link #equals(ExponentialHistogram, ExponentialHistogram)}.
+     * @param histogram the histogram to hash
+     * @return the hash code
+     */
+    static int hashCode(ExponentialHistogram histogram) {
+        int hash = histogram.scale();
+        hash = 31 * hash + Double.hashCode(histogram.sum());
+        hash = 31 * hash + Long.hashCode(histogram.valueCount());
+        hash = 31 * hash + Double.hashCode(histogram.min());
+        hash = 31 * hash + histogram.zeroBucket().hashCode();
+        // we intentionally don't include the hash of the buckets here, because that is likely expensive to compute
+        // instead, we assume that the value count and sum are a good enough approximation in most cases to minimize collisions
+        // the value count is typically available as a cached value and doesn't involve iterating over all buckets
+        return hash;
+    }
+
     static ExponentialHistogram empty() {
         return EmptyExponentialHistogram.INSTANCE;
     }