PYTHON-3048 Fixed bug with incorrect validation of UTF-8 regex patterns (#970)

Author: bcwarner

THIRD-PARTY-NOTICES

@@ -71,26 +71,3 @@ OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-3) License Notice for encoding_helpers.c
-----------------------------------------
-
-Portions Copyright 2001 Unicode, Inc.
-
-Disclaimer
-
-This source code is provided as is by Unicode, Inc. No claims are
-made as to fitness for any particular purpose. No warranties of any
-kind are expressed or implied. The recipient agrees to determine
-applicability of information provided. If this file has been
-purchased on magnetic or optical media from Unicode, Inc., the
-sole remedy for any claim will be exchange of defective media
-within 90 days of receipt.
-
-Limitations on Rights to Redistribute This Code
-
-Unicode, Inc. hereby grants the right to freely use the information
-supplied in this file in the creation of products supporting the
-Unicode Standard, and to make copies of this file in any form
-for internal or external distribution as long as this notice
-remains attached.

bson/_cbsonmodule.c

@@ -26,7 +26,6 @@
 
 #include "buffer.h"
 #include "time64.h"
-#include "encoding_helpers.h"
 
 #define _CBSON_MODULE
 #include "_cbsonmodule.h"
@@ -553,12 +552,12 @@ static int _write_regex_to_buffer(
     PyObject* py_flags;
     PyObject* py_pattern;
     PyObject* encoded_pattern;
+    PyObject* decoded_pattern;
     long int_flags;
     char flags[FLAGS_SIZE];
     char check_utf8 = 0;
     const char* pattern_data;
     int pattern_length, flags_length;
-    result_t status;
 
     /*
      * Both the builtin re type and our Regex class have attributes
@@ -597,18 +596,8 @@ static int _write_regex_to_buffer(
         Py_DECREF(encoded_pattern);
         return 0;
     }
-    status = cbson_check_string((const unsigned char*)pattern_data,
-                          pattern_length, check_utf8, 1);
-    if (status == NOT_UTF_8) {
-        PyObject* InvalidStringData = _error("InvalidStringData");
-        if (InvalidStringData) {
-            PyErr_SetString(InvalidStringData,
-                            "regex patterns must be valid UTF-8");
-            Py_DECREF(InvalidStringData);
-        }
-        Py_DECREF(encoded_pattern);
-        return 0;
-    } else if (status == HAS_NULL) {
+
+    if (strlen(pattern_data) != (size_t) pattern_length){
         PyObject* InvalidDocument = _error("InvalidDocument");
         if (InvalidDocument) {
             PyErr_SetString(InvalidDocument,
@@ -619,6 +608,22 @@ static int _write_regex_to_buffer(
         return 0;
     }
 
+    if (check_utf8) {
+        decoded_pattern = PyUnicode_DecodeUTF8(pattern_data, (Py_ssize_t) pattern_length, NULL);
+        if (decoded_pattern == NULL) {
+            PyErr_Clear();
+            PyObject* InvalidStringData = _error("InvalidStringData");
+            if (InvalidStringData) {
+                PyErr_SetString(InvalidStringData,
+                                "regex patterns must be valid UTF-8");
+                Py_DECREF(InvalidStringData);
+            }
+            Py_DECREF(encoded_pattern);
+            return 0;
+        }
+        Py_DECREF(decoded_pattern);
+    }
+
     if (!buffer_write_bytes(buffer, pattern_data, pattern_length + 1)) {
         Py_DECREF(encoded_pattern);
         return 0;

bson/encoding_helpers.c

@@ -19,6 +19,8 @@ Bug fixes
 
 - Fixed a bug where :meth:`~pymongo.collection.Collection.estimated_document_count`
   would fail with a "CommandNotSupportedOnView" error on views (`PYTHON-2885`_).
+- Fixed a bug where invalid UTF-8 strings could be passed as patterns for :class:`~bson.regex.Regex`
+  objects (`PYTHON-3048`_). :func:`bson.encode` now correctly raises :class:`bson.errors.InvalidStringData`.
 
 Unavoidable breaking changes
 ............................
@@ -38,6 +40,7 @@ Issues Resolved
 See the `PyMongo 4.2 release notes in JIRA`_ for the list of resolved issues
 in this release.
 
+.. _PYTHON-3048: https://jira.mongodb.org/browse/PYTHON-3048
 .. _PYTHON-2885: https://jira.mongodb.org/browse/PYTHON-2885
 .. _PYTHON-3167: https://jira.mongodb.org/browse/PYTHON-3167
 .. _PyMongo 4.2 release notes in JIRA: https://jira.mongodb.org/secure/ReleaseNote.jspa?projectId=10004&version=33196

bson/encoding_helpers.h

@@ -255,12 +255,7 @@ def build_extension(self, ext):
     Extension(
         "bson._cbson",
         include_dirs=["bson"],
-        sources=[
-            "bson/_cbsonmodule.c",
-            "bson/time64.c",
-            "bson/buffer.c",
-            "bson/encoding_helpers.c",
-        ],
+        sources=["bson/_cbsonmodule.c", "bson/time64.c", "bson/buffer.c"],
     ),
     Extension(
         "pymongo._cmessage",