tree data BUGFIX check for valid UTF-8 strings

michalvasko · michalvasko · commit 989cdb483436 · 2023-10-06T15:35:19.000+02:00
Fixes sysrepo/sysrepo#3173
diff --git a/src/common.c b/src/common.c
@@ -247,6 +247,157 @@ ly_getutf8(const char **input, uint32_t *utf8_char, size_t *bytes_read)
     return LY_SUCCESS;
 }
 
+/**
+ * @brief Check whether an UTF-8 string is equal to a hex string after a bitwise and.
+ *
+ * (input & 0x[arg1][arg3][arg5]...) == 0x[arg2][arg4][arg6]...
+ *
+ * @param[in] input UTF-8 string.
+ * @param[in] bytes Number of bytes to compare.
+ * @param[in] ... 2x @p bytes number of bytes to perform bitwise and and equality operations.
+ * @return Result of the operation.
+ */
+static int
+ly_utf8_and_equal(const char *input, uint8_t bytes, ...)
+{
+    va_list ap;
+    int i, and, byte;
+
+    va_start(ap, bytes);
+    for (i = 0; i < bytes; ++i) {
+        and = va_arg(ap, int);
+        byte = va_arg(ap, int);
+
+        /* compare each byte */
+        if (((uint8_t)input[i] & and) != (uint8_t)byte) {
+            return 0;
+        }
+    }
+    va_end(ap);
+
+    return 1;
+}
+
+/**
+ * @brief Check whether an UTF-8 string is smaller than a hex string.
+ *
+ * input < 0x[arg1][arg2]...
+ *
+ * @param[in] input UTF-8 string.
+ * @param[in] bytes Number of bytes to compare.
+ * @param[in] ... @p bytes number of bytes to compare with.
+ * @return Result of the operation.
+ */
+static int
+ly_utf8_less(const char *input, uint8_t bytes, ...)
+{
+    va_list ap;
+    int i, byte;
+
+    va_start(ap, bytes);
+    for (i = 0; i < bytes; ++i) {
+        byte = va_arg(ap, int);
+
+        /* compare until bytes differ */
+        if ((uint8_t)input[i] > (uint8_t)byte) {
+            return 0;
+        } else if ((uint8_t)input[i] < (uint8_t)byte) {
+            return 1;
+        }
+    }
+    va_end(ap);
+
+    /* equals */
+    return 0;
+}
+
+/**
+ * @brief Check whether an UTF-8 string is greater than a hex string.
+ *
+ * input > 0x[arg1][arg2]...
+ *
+ * @param[in] input UTF-8 string.
+ * @param[in] bytes Number of bytes to compare.
+ * @param[in] ... @p bytes number of bytes to compare with.
+ * @return Result of the operation.
+ */
+static int
+ly_utf8_greater(const char *input, uint8_t bytes, ...)
+{
+    va_list ap;
+    int i, byte;
+
+    va_start(ap, bytes);
+    for (i = 0; i < bytes; ++i) {
+        byte = va_arg(ap, int);
+
+        /* compare until bytes differ */
+        if ((uint8_t)input[i] > (uint8_t)byte) {
+            return 1;
+        } else if ((uint8_t)input[i] < (uint8_t)byte) {
+            return 0;
+        }
+    }
+    va_end(ap);
+
+    /* equals */
+    return 0;
+}
+
+LY_ERR
+ly_checkutf8(const char *input, size_t in_len, size_t *utf8_len)
+{
+    size_t len;
+
+    if (!(input[0] & 0x80)) {
+        /* one byte character */
+        len = 1;
+
+        if (ly_utf8_less(input, 1, 0x20) && (input[0] != 0x9) && (input[0] != 0xa) && (input[0] != 0xd)) {
+            /* invalid control characters */
+            return LY_EINVAL;
+        }
+    } else if (((input[0] & 0xe0) == 0xc0) && (in_len > 1)) {
+        /* two bytes character */
+        len = 2;
+
+        /* (input < 0xC280) || (input > 0xDFBF) || ((input & 0xE0C0) != 0xC080) */
+        if (ly_utf8_less(input, 2, 0xC2, 0x80) || ly_utf8_greater(input, 2, 0xDF, 0xBF) ||
+                !ly_utf8_and_equal(input, 2, 0xE0, 0xC0, 0xC0, 0x80)) {
+            return LY_EINVAL;
+        }
+    } else if (((input[0] & 0xf0) == 0xe0) && (in_len > 2)) {
+        /* three bytes character */
+        len = 3;
+
+        /* (input >= 0xEDA080) && (input <= 0xEDBFBF) */
+        if (!ly_utf8_less(input, 3, 0xED, 0xA0, 0x80) && !ly_utf8_greater(input, 3, 0xED, 0xBF, 0xBF)) {
+            /* reject UTF-16 surrogates */
+            return LY_EINVAL;
+        }
+
+        /* (input < 0xE0A080) || (input > 0xEFBFBF) || ((input & 0xF0C0C0) != 0xE08080) */
+        if (ly_utf8_less(input, 3, 0xE0, 0xA0, 0x80) || ly_utf8_greater(input, 3, 0xEF, 0xBF, 0xBF) ||
+                !ly_utf8_and_equal(input, 3, 0xF0, 0xE0, 0xC0, 0x80, 0xC0, 0x80)) {
+            return LY_EINVAL;
+        }
+    } else if (((input[0] & 0xf8) == 0xf0) && (in_len > 3)) {
+        /* four bytes character */
+        len = 4;
+
+        /* (input < 0xF0908080) || (input > 0xF48FBFBF) || ((input & 0xF8C0C0C0) != 0xF0808080) */
+        if (ly_utf8_less(input, 4, 0xF0, 0x90, 0x80, 0x80) || ly_utf8_greater(input, 4, 0xF4, 0x8F, 0xBF, 0xBF) ||
+                !ly_utf8_and_equal(input, 4, 0xF8, 0xF0, 0xC0, 0x80, 0xC0, 0x80, 0xC0, 0x80)) {
+            return LY_EINVAL;
+        }
+    } else {
+        return LY_EINVAL;
+    }
+
+    *utf8_len = len;
+    return LY_SUCCESS;
+}
+
 LY_ERR
 ly_pututf8(char *dst, uint32_t value, size_t *bytes_written)
 {
diff --git a/src/common.h b/src/common.h
@@ -539,6 +539,17 @@ LY_ERR ly_value_prefix_next(const char *str_begin, const char *str_end, uint32_t
  */
 LY_ERR ly_getutf8(const char **input, uint32_t *utf8_char, size_t *bytes_read);
 
+/**
+ * @brief Check an UTF-8 character is valid.
+ *
+ * @param[in] input Input string to process.
+ * @param[in] in_len Bytes left to read in @p input.
+ * @param[out] utf8_len Length of a valid UTF-8 character.
+ * @return LY_SUCCESS on success
+ * @return LY_EINVAL in case of invalid UTF-8 character.
+ */
+LY_ERR ly_checkutf8(const char *input, size_t in_len, size_t *utf8_len);
+
 /**
  * @brief Store UTF-8 character specified as 4byte integer into the dst buffer.
  *
diff --git a/src/parser_common.c b/src/parser_common.c
@@ -285,7 +285,7 @@ lyd_parser_create_term(struct lyd_ctx *lydctx, const struct lysc_node *schema, c
     LY_ERR r;
     ly_bool incomplete;
 
-    if ((r = lyd_create_term(schema, value, value_len, dynamic, format, prefix_data, hints, &incomplete, node))) {
+    if ((r = lyd_create_term(schema, value, value_len, 1, dynamic, format, prefix_data, hints, &incomplete, node))) {
         if (lydctx->data_ctx->ctx != schema->module->ctx) {
             /* move errors to the main context */
             ly_err_move(schema->module->ctx, (struct ly_ctx *)lydctx->data_ctx->ctx);
@@ -323,8 +323,8 @@ lyd_parser_create_meta(struct lyd_ctx *lydctx, struct lyd_node *parent, struct l
     }
     LOG_LOCSET(NULL, NULL, path, NULL);
 
-    LY_CHECK_GOTO(rc = lyd_create_meta(parent, meta, mod, name, name_len, value, value_len, dynamic, format, prefix_data,
-            hints, ctx_node, 0, &incomplete), cleanup);
+    LY_CHECK_GOTO(rc = lyd_create_meta(parent, meta, mod, name, name_len, value, value_len, 1, dynamic, format,
+            prefix_data, hints, ctx_node, 0, &incomplete), cleanup);
 
     if (incomplete && !(lydctx->parse_opts & LYD_PARSE_ONLY)) {
         LY_CHECK_GOTO(rc = ly_set_add(&lydctx->meta_types, *meta, 1, NULL), cleanup);
diff --git a/src/path.c b/src/path.c
@@ -698,8 +698,8 @@ ly_path_compile_predicate(const struct ly_ctx *ctx, const struct lysc_node *cur_
 
                 /* store the value */
                 LOG_LOCSET(key, NULL, NULL, NULL);
-                ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaf *)key)->type, val, val_len, NULL, format,
-                        prefix_data, LYD_HINT_DATA, key, NULL);
+                ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaf *)key)->type, val, val_len, 0, NULL,
+                        format, prefix_data, LYD_HINT_DATA, key, NULL);
                 LOG_LOCBACK(key ? 1 : 0, 0, 0, 0);
                 LY_CHECK_ERR_GOTO(ret, p->value.realtype = NULL, cleanup);
 
@@ -762,8 +762,8 @@ ly_path_compile_predicate(const struct ly_ctx *ctx, const struct lysc_node *cur_
 
         /* store the value */
         LOG_LOCSET(ctx_node, NULL, NULL, NULL);
-        ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaflist *)ctx_node)->type, val, val_len, NULL, format,
-                prefix_data, LYD_HINT_DATA, ctx_node, NULL);
+        ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaflist *)ctx_node)->type, val, val_len, 0, NULL,
+                format, prefix_data, LYD_HINT_DATA, ctx_node, NULL);
         LOG_LOCBACK(ctx_node ? 1 : 0, 0, 0, 0);
         LY_CHECK_ERR_GOTO(ret, p->value.realtype = NULL, cleanup);
         ++(*tok_idx);
diff --git a/src/plugins_types.h b/src/plugins_types.h
@@ -473,6 +473,7 @@ LIBYANG_API_DECL LY_ERR lyplg_type_print_xpath10_value(const struct lyd_value_xp
                                              value after calling the type's store callback with this option. */
 #define LYPLG_TYPE_STORE_IMPLEMENT 0x02 /**< If a foreign module is needed to be implemented to successfully instantiate
                                              the value, make the module implemented. */
+#define LYPLG_TYPE_STORE_IS_UTF8   0x04 /**< The value is guaranteed to be a valid UTF-8 string, if applicable for the type. */
 /** @} plugintypestoreopts */
 
 /**
diff --git a/src/plugins_types/string.c b/src/plugins_types/string.c
@@ -1,9 +1,10 @@
 /**
  * @file string.c
  * @author Radek Krejci <rkrejci@cesnet.cz>
+ * @author Michal Vasko <mvasko@cesnet.cz>
  * @brief Built-in string type plugin.
  *
- * Copyright (c) 2019-2021 CESNET, z.s.p.o.
+ * Copyright (c) 2019 - 2023 CESNET, z.s.p.o.
  *
  * This source code is licensed under BSD 3-Clause License (the "License").
  * You may not use this file except in compliance with the License.
@@ -33,6 +34,29 @@
  * | string length | yes | `char *` | string itself |
  */
 
+/**
+ * @brief Check string value for invalid characters.
+ *
+ * @param[in] value String to check.
+ * @param[in] value_len Length of @p value.
+ * @param[out] err Generated error on error.
+ * @return LY_ERR value.
+ */
+static LY_ERR
+string_check_chars(const char *value, size_t value_len, struct ly_err_item **err)
+{
+    size_t len, parsed = 0;
+
+    while (value_len - parsed) {
+        if (ly_checkutf8(value + parsed, value_len - parsed, &len)) {
+            return ly_err_new(err, LY_EVALID, LYVE_DATA, NULL, NULL, "Invalid character 0x%hhx.", value[parsed]);
+        }
+        parsed += len;
+    }
+
+    return LY_SUCCESS;
+}
+
 LIBYANG_API_DEF LY_ERR
 lyplg_type_store_string(const struct ly_ctx *ctx, const struct lysc_type *type, const void *value, size_t value_len,
         uint32_t options, LY_VALUE_FORMAT UNUSED(format), void *UNUSED(prefix_data), uint32_t hints,
@@ -46,6 +70,12 @@ lyplg_type_store_string(const struct ly_ctx *ctx, const struct lysc_type *type,
     memset(storage, 0, sizeof *storage);
     storage->realtype = type;
 
+    if (!(options & LYPLG_TYPE_STORE_IS_UTF8)) {
+        /* check the UTF-8 encoding */
+        ret = string_check_chars(value, value_len, err);
+        LY_CHECK_GOTO(ret, cleanup);
+    }
+
     /* check hints */
     ret = lyplg_type_check_hints(hints, value, value_len, type->basetype, NULL, err);
     LY_CHECK_GOTO(ret, cleanup);
diff --git a/src/tree_data.c b/src/tree_data.c
@@ -991,7 +991,7 @@ lyd_insert_meta(struct lyd_node *parent, struct lyd_meta *meta, ly_bool clear_df
 
 LY_ERR
 lyd_create_meta(struct lyd_node *parent, struct lyd_meta **meta, const struct lys_module *mod, const char *name,
-        size_t name_len, const char *value, size_t value_len, ly_bool *dynamic, LY_VALUE_FORMAT format,
+        size_t name_len, const char *value, size_t value_len, ly_bool is_utf8, ly_bool *dynamic, LY_VALUE_FORMAT format,
         void *prefix_data, uint32_t hints, const struct lysc_node *ctx_node, ly_bool clear_dflt, ly_bool *incomplete)
 {
     LY_ERR ret = LY_SUCCESS;
@@ -1023,7 +1023,7 @@ lyd_create_meta(struct lyd_node *parent, struct lyd_meta **meta, const struct ly
     mt->parent = parent;
     mt->annotation = ant;
     lyplg_ext_get_storage(ant, LY_STMT_TYPE, sizeof ant_type, (const void **)&ant_type);
-    ret = lyd_value_store(mod->ctx, &mt->value, ant_type, value, value_len, dynamic, format, prefix_data, hints,
+    ret = lyd_value_store(mod->ctx, &mt->value, ant_type, value, value_len, is_utf8, dynamic, format, prefix_data, hints,
             ctx_node, incomplete);
     LY_CHECK_ERR_GOTO(ret, free(mt), cleanup);
     ret = lydict_insert(mod->ctx, name, name_len, &mt->name);
@@ -1798,7 +1798,7 @@ lyd_dup_r(const struct lyd_node *node, const struct ly_ctx *trg_ctx, struct lyd_
             /* store canonical value in the target context */
             val_can = lyd_get_value(node);
             type = ((struct lysc_node_leaf *)term->schema)->type;
-            ret = lyd_value_store(trg_ctx, &term->value, type, val_can, strlen(val_can), NULL, LY_VALUE_CANON, NULL,
+            ret = lyd_value_store(trg_ctx, &term->value, type, val_can, strlen(val_can), 1, NULL, LY_VALUE_CANON, NULL,
                     LYD_HINT_DATA, term->schema, NULL);
             LY_CHECK_GOTO(ret, error);
         }
@@ -2727,7 +2727,7 @@ lyd_find_sibling_val(const struct lyd_node *siblings, const struct lysc_node *sc
         /* create a data node and find the instance */
         if (schema->nodetype == LYS_LEAFLIST) {
             /* target used attributes: schema, hash, value */
-            rc = lyd_create_term(schema, key_or_value, val_len, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, NULL, &target);
+            rc = lyd_create_term(schema, key_or_value, val_len, 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, NULL, &target);
             LY_CHECK_RET(rc);
         } else {
             /* target used attributes: schema, hash, child (all keys) */
diff --git a/src/tree_data_common.c b/src/tree_data_common.c
@@ -476,12 +476,12 @@ lyd_data_next_module(struct lyd_node **next, struct lyd_node **first)
 
 LY_ERR
 lyd_value_store(const struct ly_ctx *ctx, struct lyd_value *val, const struct lysc_type *type, const void *value,
-        size_t value_len, ly_bool *dynamic, LY_VALUE_FORMAT format, void *prefix_data, uint32_t hints,
+        size_t value_len, ly_bool is_utf8, ly_bool *dynamic, LY_VALUE_FORMAT format, void *prefix_data, uint32_t hints,
         const struct lysc_node *ctx_node, ly_bool *incomplete)
 {
     LY_ERR ret;
     struct ly_err_item *err = NULL;
-    uint32_t options = (dynamic && *dynamic ? LYPLG_TYPE_STORE_DYNAMIC : 0);
+    uint32_t options = 0;
 
     if (!value) {
         value = "";
@@ -490,6 +490,13 @@ lyd_value_store(const struct ly_ctx *ctx, struct lyd_value *val, const struct ly
         *incomplete = 0;
     }
 
+    if (dynamic && *dynamic) {
+        options |= LYPLG_TYPE_STORE_DYNAMIC;
+    }
+    if (is_utf8) {
+        options |= LYPLG_TYPE_STORE_IS_UTF8;
+    }
+
     ret = type->plugin->store(ctx, type, value, value_len, options, format, prefix_data, hints, ctx_node, val, NULL, &err);
     if (dynamic) {
         *dynamic = 0;
@@ -676,7 +683,7 @@ lyd_value_compare(const struct lyd_node_term *node, const char *value, size_t va
 
     /* store the value */
     LOG_LOCSET(node->schema, &node->node, NULL, NULL);
-    ret = lyd_value_store(ctx, &val, type, value, value_len, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, node->schema, NULL);
+    ret = lyd_value_store(ctx, &val, type, value, value_len, 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, node->schema, NULL);
     LOG_LOCBACK(1, 1, 0, 0);
     LY_CHECK_RET(ret);
 
diff --git a/src/tree_data_internal.h b/src/tree_data_internal.h
diff --git a/src/tree_data_new.c b/src/tree_data_new.c
