access string internals via functions

ngoldbaum · ngoldbaum · commit 74e97fd4d7aa · 2023-09-14T14:40:58.000-06:00
diff --git a/stringdtype/stringdtype/src/casts.c b/stringdtype/stringdtype/src/casts.c
@@ -213,7 +213,7 @@ unicode_to_string(PyArrayMethod_Context *context, char *const data[],
             gil_error(PyExc_MemoryError, "npy_string_newemptysize failed");
             return -1;
         }
-        char *out_buf = out_ss->buf;
+        char *out_buf = npy_string_buf(out_ss);
         for (size_t i = 0; i < num_codepoints; i++) {
             // get code point
             Py_UCS4 code = in[i];
@@ -338,28 +338,28 @@ string_to_unicode(PyArrayMethod_Context *context, char *const data[],
         s = (npy_static_string *)in;
         unsigned char *this_string = NULL;
         size_t n_bytes;
+        const npy_static_string *name = NULL;
         if (npy_string_isnull(s)) {
             if (has_null && !has_string_na) {
                 // lossy but not much else we can do
-                this_string = (unsigned char *)descr->na_name.buf;
-                n_bytes = descr->na_name.size;
+                name = &descr->na_name;
             }
             else {
-                this_string = (unsigned char *)(default_string.buf);
-                n_bytes = default_string.size;
+                name = &default_string;
             }
         }
         else {
-            this_string = (unsigned char *)(s->buf);
-            n_bytes = s->size;
+            name = s;
         }
+
+        this_string = (unsigned char *)npy_string_buf(name);
+        n_bytes = npy_string_size(name);
         size_t tot_n_bytes = 0;
 
         for (int i = 0; i < max_out_size; i++) {
             Py_UCS4 code;
 
-            // get code point for character this_string is currently pointing
-            // too
+            // code point for character this_string is currently pointing at
             size_t num_bytes =
                     utf8_char_to_ucs4_code(this_string, n_bytes, &code);
 
@@ -440,10 +440,10 @@ string_to_bool(PyArrayMethod_Context *context, char *const data[],
                 *out = (npy_bool)1;
             }
             else {
-                *out = (npy_bool)(default_string.size == 0);
+                *out = (npy_bool)(npy_string_size(&default_string) == 0);
             }
         }
-        else if (s->size == 0) {
+        else if (npy_string_size(s) == 0) {
             *out = (npy_bool)0;
         }
         else {
@@ -527,7 +527,8 @@ string_to_pylong(char *in, int hasnull)
         }
         s = &EMPTY_STRING;
     }
-    PyObject *val_obj = PyUnicode_FromStringAndSize(s->buf, s->size);
+    PyObject *val_obj =
+            PyUnicode_FromStringAndSize(npy_string_buf(s), npy_string_size(s));
     if (val_obj == NULL) {
         return NULL;
     }
@@ -779,7 +780,8 @@ string_to_pyfloat(char *in, int hasnull)
         }
         s = &EMPTY_STRING;
     }
-    PyObject *val_obj = PyUnicode_FromStringAndSize(s->buf, s->size);
+    PyObject *val_obj =
+            PyUnicode_FromStringAndSize(npy_string_buf(s), npy_string_size(s));
     if (val_obj == NULL) {
         return NULL;
     }
@@ -1004,8 +1006,9 @@ string_to_datetime(PyArrayMethod_Context *context, char *const data[],
             s = &default_string;
         }
         if (NpyDatetime_ParseISO8601Datetime(
-                    (const char *)s->buf, s->size, in_unit, NPY_UNSAFE_CASTING,
-                    &dts, &in_meta.base, &out_special) < 0) {
+                    (const char *)npy_string_buf(s), npy_string_size(s),
+                    in_unit, NPY_UNSAFE_CASTING, &dts, &in_meta.base,
+                    &out_special) < 0) {
             return -1;
         }
         if (NpyDatetime_ConvertDatetimeStructToDatetime64(dt_meta, &dts, out) <
diff --git a/stringdtype/stringdtype/src/dtype.c b/stringdtype/stringdtype/src/dtype.c
@@ -223,7 +223,6 @@ stringdtype_setitem(StringDTypeObject *descr, PyObject *obj, char **dataptr)
             return -1;
         }
 
-        // copies contents of val into item_val->buf
         int res = npy_string_newsize(val, length, sdata);
 
         if (res == -1) {
@@ -260,8 +259,8 @@ stringdtype_getitem(StringDTypeObject *descr, char **dataptr)
         }
     }
     else {
-        char *data = sdata->buf;
-        size_t size = sdata->size;
+        char *data = npy_string_buf(sdata);
+        size_t size = npy_string_size(sdata);
         val_obj = PyUnicode_FromStringAndSize(data, size);
         if (val_obj == NULL) {
             return NULL;
@@ -287,7 +286,7 @@ stringdtype_getitem(StringDTypeObject *descr, char **dataptr)
 npy_bool
 nonzero(void *data, void *NPY_UNUSED(arr))
 {
-    return ((npy_static_string *)data)->size != 0;
+    return npy_string_size((npy_static_string *)data) != 0;
 }
 
 // Implementation of PyArray_CompareFunc.
diff --git a/stringdtype/stringdtype/src/main.c b/stringdtype/stringdtype/src/main.c
@@ -58,7 +58,7 @@ _memory_usage(PyObject *NPY_UNUSED(self), PyObject *obj)
         npy_intp count = *innersizeptr;
 
         while (count--) {
-            memory_usage += ((npy_static_string *)in)->size;
+            memory_usage += npy_string_size(((npy_static_string *)in));
             in += stride;
         }
 
diff --git a/stringdtype/stringdtype/src/static_string.c b/stringdtype/stringdtype/src/static_string.c
@@ -11,7 +11,8 @@ const npy_static_string NULL_STRING = {0, NULL};
 int
 npy_string_newsize(const char *init, size_t size, npy_static_string *to_init)
 {
-    if ((to_init == NULL) || (to_init->buf != NULL) || (to_init->size != 0)) {
+    if ((to_init == NULL) || (to_init->buf != NULL) ||
+        (npy_string_size(to_init) != 0)) {
         return -2;
     }
 
@@ -110,3 +111,24 @@ npy_string_isnull(const npy_static_string *in)
     }
     return 0;
 }
+
+size_t
+npy_string_size(const npy_static_string *s)
+{
+    return s->size;
+}
+
+char *
+npy_string_buf(const npy_static_string *s)
+{
+    return s->buf;
+}
+
+int
+npy_string_size_and_buf(const npy_static_string *s, size_t *size, char **buf)
+{
+    *size = s->size;
+    *buf = s->buf;
+
+    return 0;
+}
diff --git a/stringdtype/stringdtype/src/static_string.h b/stringdtype/stringdtype/src/static_string.h
@@ -17,15 +17,18 @@ extern const npy_static_string EMPTY_STRING;
 // working with it.
 extern const npy_static_string NULL_STRING;
 
-// Allocates a new buffer for *to_init*, filling with the copied contents of
-// the first *size*entries in *init*, which must be valid and initialized
-// beforehand, and sets *to_init->size* to *size*. Returns -1 if malloc fails
-// and -2 if *to_init* is not NULL. Returns 0 on success.
+// Allocates a new buffer for *to_init*, which must be set to NULL before
+// calling this function, filling the newly allocated buffer with the copied
+// contents of the first *size* entries in *init*, which must be valid and
+// initialized beforehand. Calling npy_string_free on *to_init* before calling
+// this function on an existing string is sufficient to initialize it. Returns
+// -1 if malloc fails and -2 if the internal buffer in *to_init* is not NULL
+// to indicate a programming error. Returns 0 on success.
 int
 npy_string_newsize(const char *init, size_t size, npy_static_string *to_init);
 
-// Sets len to 0 and if str->buf is not already NULL, frees it and sets it to
-// NULL. Cannot fail.
+// Sets len to 0 and if the internal buffer is not already NULL, frees it if
+// it is allocated on the heap and sets it to NULL. Cannot fail.
 void
 npy_string_free(npy_static_string *str);
 
@@ -54,4 +57,17 @@ npy_string_isnull(const npy_static_string *in);
 int
 npy_string_cmp(const npy_static_string *s1, const npy_static_string *s2);
 
+// Returns the *size* of *s*
+size_t
+npy_string_size(const npy_static_string *s);
+
+// Returns the string *buf* of *s*. This is not a null-terminated buffer.
+char *
+npy_string_buf(const npy_static_string *s);
+
+// Fills in *size* and *buf* pointers with the values in *s*.
+// Currently always returns 0.
+int
+npy_string_size_and_buf(const npy_static_string *s, size_t *size, char **buf);
+
 #endif /*_NPY_STATIC_STRING_H */
diff --git a/stringdtype/stringdtype/src/umath.c b/stringdtype/stringdtype/src/umath.c
@@ -67,7 +67,9 @@ multiply_resolve_descriptors(
                 }                                                             \
             }                                                                 \
             npy_##shortname factor = *(npy_##shortname *)iin;                 \
-            size_t newsize = (size_t)((is->size) * factor);                   \
+            size_t cursize = npy_string_size(is);                             \
+            /* FIXME: check for overflow? */                                  \
+            size_t newsize = cursize * factor;                                \
                                                                               \
             if (npy_string_newemptysize(newsize, os) < 0) {                   \
                 gil_error(PyExc_MemoryError,                                  \
@@ -76,7 +78,8 @@ multiply_resolve_descriptors(
             }                                                                 \
                                                                               \
             for (size_t i = 0; i < (size_t)factor; i++) {                     \
-                memcpy(os->buf + i * is->size, is->buf, is->size);            \
+                memcpy(npy_string_buf(os) + i * cursize, npy_string_buf(is),  \
+                       cursize);                                              \
             }                                                                 \
                                                                               \
             sin += s_stride;                                                  \
@@ -215,7 +218,6 @@ add_strided_loop(PyArrayMethod_Context *context, char *const data[],
     npy_static_string *os = NULL;
 
     while (N--) {
-        int newsize = 0;
         s1 = (npy_static_string *)in1;
         s2 = (npy_static_string *)in2;
         int s1_isnull = npy_string_isnull(s1);
@@ -240,13 +242,20 @@ add_strided_loop(PyArrayMethod_Context *context, char *const data[],
                           "Cannot add null that is not a nan-like value");
             }
         }
-        newsize = s1->size + s2->size;
-        if (npy_string_newemptysize(newsize, os) < 0) {
+
+        size_t s1_size = npy_string_size(s1);
+        size_t s2_size = npy_string_size(s2);
+
+        if (npy_string_newemptysize(s1_size + s2_size, os) < 0) {
             return -1;
         }
 
-        memcpy(os->buf, s1->buf, s1->size);
-        memcpy(os->buf + s1->size, s2->buf, s2->size);
+        char *os_buf = npy_string_buf(os);
+        char *s1_buf = npy_string_buf(s1);
+        char *s2_buf = npy_string_buf(s2);
+
+        memcpy(os_buf, s1_buf, s1_size);
+        memcpy(os_buf + s1_size, s2_buf, s2_size);
 
     next_step:
         in1 += in1_stride;
@@ -385,7 +394,11 @@ string_equal_strided_loop(PyArrayMethod_Context *context, char *const data[],
                 }
             }
         }
-        if (s1->size == s2->size && strncmp(s1->buf, s2->buf, s1->size) == 0) {
+        char *s1_buf = npy_string_buf(s1);
+        char *s2_buf = npy_string_buf(s2);
+        size_t s1_size = npy_string_size(s1);
+        size_t s2_size = npy_string_size(s2);
+        if (s1_size == s2_size && strncmp(s1_buf, s2_buf, s1_size) == 0) {
             *out = (npy_bool)1;
         }
         else {
@@ -450,7 +463,14 @@ string_not_equal_strided_loop(PyArrayMethod_Context *context,
                 }
             }
         }
-        if (s1->size == s2->size && strncmp(s1->buf, s2->buf, s1->size) == 0) {
+
+        size_t s1_size = npy_string_size(s1);
+        size_t s2_size = npy_string_size(s2);
+
+        char *s1_buf = npy_string_buf(s1);
+        char *s2_buf = npy_string_buf(s2);
+
+        if (s1_size == s2_size && strncmp(s1_buf, s2_buf, s1_size) == 0) {
             *out = (npy_bool)0;
         }
         else {

Original file line number	Diff line number	Diff line change
`@@ -213,7 +213,7 @@ unicode_to_string(PyArrayMethod_Context context, char const data[],`
`213`	`213`	`gil_error(PyExc_MemoryError, "npy_string_newemptysize failed");`
`214`	`214`	`return -1;`
`215`	`215`	`}`
`216`		`- char *out_buf = out_ss->buf;`
	`216`	`+ char *out_buf = npy_string_buf(out_ss);`
`217`	`217`	`for (size_t i = 0; i < num_codepoints; i++) {`
`218`	`218`	`// get code point`
`219`	`219`	`Py_UCS4 code = in[i];`
`@@ -338,28 +338,28 @@ string_to_unicode(PyArrayMethod_Context context, char const data[],`
`338`	`338`	`s = (npy_static_string *)in;`
`339`	`339`	`unsigned char *this_string = NULL;`
`340`	`340`	`size_t n_bytes;`
	`341`	`+ const npy_static_string *name = NULL;`
`341`	`342`	`if (npy_string_isnull(s)) {`
`342`	`343`	`if (has_null && !has_string_na) {`
`343`	`344`	`// lossy but not much else we can do`
`344`		`- this_string = (unsigned char *)descr->na_name.buf;`
`345`		`- n_bytes = descr->na_name.size;`
	`345`	`+ name = &descr->na_name;`
`346`	`346`	`}`
`347`	`347`	`else {`
`348`		`- this_string = (unsigned char *)(default_string.buf);`
`349`		`- n_bytes = default_string.size;`
	`348`	`+ name = &default_string;`
`350`	`349`	`}`
`351`	`350`	`}`
`352`	`351`	`else {`
`353`		`- this_string = (unsigned char *)(s->buf);`
`354`		`- n_bytes = s->size;`
	`352`	`+ name = s;`
`355`	`353`	`}`
	`354`	`+`
	`355`	`+ this_string = (unsigned char *)npy_string_buf(name);`
	`356`	`+ n_bytes = npy_string_size(name);`
`356`	`357`	`size_t tot_n_bytes = 0;`
`357`	`358`
`358`	`359`	`for (int i = 0; i < max_out_size; i++) {`
`359`	`360`	`Py_UCS4 code;`
`360`	`361`
`361`		`- // get code point for character this_string is currently pointing`
`362`		`- // too`
	`362`	`+ // code point for character this_string is currently pointing at`
`363`	`363`	`size_t num_bytes =`
`364`	`364`	`utf8_char_to_ucs4_code(this_string, n_bytes, &code);`
`365`	`365`
`@@ -440,10 +440,10 @@ string_to_bool(PyArrayMethod_Context context, char const data[],`
`440`	`440`	`*out = (npy_bool)1;`
`441`	`441`	`}`
`442`	`442`	`else {`
`443`		`- *out = (npy_bool)(default_string.size == 0);`
	`443`	`+ *out = (npy_bool)(npy_string_size(&default_string) == 0);`
`444`	`444`	`}`
`445`	`445`	`}`
`446`		`- else if (s->size == 0) {`
	`446`	`+ else if (npy_string_size(s) == 0) {`
`447`	`447`	`*out = (npy_bool)0;`
`448`	`448`	`}`
`449`	`449`	`else {`
`@@ -527,7 +527,8 @@ string_to_pylong(char *in, int hasnull)`
`527`	`527`	`}`
`528`	`528`	`s = &EMPTY_STRING;`
`529`	`529`	`}`
`530`		`- PyObject *val_obj = PyUnicode_FromStringAndSize(s->buf, s->size);`
	`530`	`+ PyObject *val_obj =`
	`531`	`+ PyUnicode_FromStringAndSize(npy_string_buf(s), npy_string_size(s));`
`531`	`532`	`if (val_obj == NULL) {`
`532`	`533`	`return NULL;`
`533`	`534`	`}`
`@@ -779,7 +780,8 @@ string_to_pyfloat(char *in, int hasnull)`
`779`	`780`	`}`
`780`	`781`	`s = &EMPTY_STRING;`
`781`	`782`	`}`
`782`		`- PyObject *val_obj = PyUnicode_FromStringAndSize(s->buf, s->size);`
	`783`	`+ PyObject *val_obj =`
	`784`	`+ PyUnicode_FromStringAndSize(npy_string_buf(s), npy_string_size(s));`
`783`	`785`	`if (val_obj == NULL) {`
`784`	`786`	`return NULL;`
`785`	`787`	`}`
`@@ -1004,8 +1006,9 @@ string_to_datetime(PyArrayMethod_Context context, char const data[],`
`1004`	`1006`	`s = &default_string;`
`1005`	`1007`	`}`
`1006`	`1008`	`if (NpyDatetime_ParseISO8601Datetime(`
`1007`		`- (const char *)s->buf, s->size, in_unit, NPY_UNSAFE_CASTING,`
`1008`		`- &dts, &in_meta.base, &out_special) < 0) {`
	`1009`	`+ (const char *)npy_string_buf(s), npy_string_size(s),`
	`1010`	`+ in_unit, NPY_UNSAFE_CASTING, &dts, &in_meta.base,`
	`1011`	`+ &out_special) < 0) {`
`1009`	`1012`	`return -1;`
`1010`	`1013`	`}`
`1011`	`1014`	`if (NpyDatetime_ConvertDatetimeStructToDatetime64(dt_meta, &dts, out) <`
Original file line number	Diff line number	Diff line change
`@@ -223,7 +223,6 @@ stringdtype_setitem(StringDTypeObject descr, PyObject obj, char **dataptr)`
`223`	`223`	`return -1;`
`224`	`224`	`}`
`225`	`225`
`226`		`- // copies contents of val into item_val->buf`
`227`	`226`	`int res = npy_string_newsize(val, length, sdata);`
`228`	`227`
`229`	`228`	`if (res == -1) {`
`@@ -260,8 +259,8 @@ stringdtype_getitem(StringDTypeObject descr, char *dataptr)`
`260`	`259`	`}`
`261`	`260`	`}`
`262`	`261`	`else {`
`263`		`- char *data = sdata->buf;`
`264`		`- size_t size = sdata->size;`
	`262`	`+ char *data = npy_string_buf(sdata);`
	`263`	`+ size_t size = npy_string_size(sdata);`
`265`	`264`	`val_obj = PyUnicode_FromStringAndSize(data, size);`
`266`	`265`	`if (val_obj == NULL) {`
`267`	`266`	`return NULL;`
`@@ -287,7 +286,7 @@ stringdtype_getitem(StringDTypeObject descr, char *dataptr)`
`287`	`286`	`npy_bool`
`288`	`287`	`nonzero(void data, void NPY_UNUSED(arr))`
`289`	`288`	`{`
`290`		`- return ((npy_static_string *)data)->size != 0;`
	`289`	`+ return npy_string_size((npy_static_string *)data) != 0;`
`291`	`290`	`}`
`292`	`291`
`293`	`292`	`// Implementation of PyArray_CompareFunc.`
Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,7 @@ _memory_usage(PyObject NPY_UNUSED(self), PyObject obj)`
`58`	`58`	`npy_intp count = *innersizeptr;`
`59`	`59`
`60`	`60`	`while (count--) {`
`61`		`- memory_usage += ((npy_static_string *)in)->size;`
	`61`	`+ memory_usage += npy_string_size(((npy_static_string *)in));`
`62`	`62`	`in += stride;`
`63`	`63`	`}`
`64`	`64`