perf: use a local buffer to store the processed string

Author: Alvaro-Kothe

pandas/_libs/include/pandas/parser/tokenizer.h

@@ -17,7 +17,6 @@ See LICENSE for the license
 #define ERROR_NO_DIGITS 1
 #define ERROR_OVERFLOW 2
 #define ERROR_INVALID_CHARS 3
-#define ERROR_NO_MEMORY 4
 
 #include <stdint.h>
 

pandas/_libs/parsers.pyx

@@ -149,7 +149,7 @@ cdef extern from "pandas/parser/tokenizer.h":
         SKIP_LINE
         FINISHED
 
-    enum: ERROR_OVERFLOW, ERROR_NO_MEMORY
+    enum: ERROR_OVERFLOW
 
     ctypedef enum BadLineHandleMethod:
         ERROR,
@@ -1822,8 +1822,6 @@ cdef _try_uint64(parser_t *parser, int64_t col,
         if error == ERROR_OVERFLOW:
             # Can't get the word variable
             raise OverflowError("Overflow")
-        if error == ERROR_NO_MEMORY:
-            raise MemoryError()
         return None
 
     if uint64_conflict(&state):
@@ -1894,8 +1892,6 @@ cdef _try_int64(parser_t *parser, int64_t col,
         if error == ERROR_OVERFLOW:
             # Can't get the word variable
             raise OverflowError("Overflow")
-        if error == ERROR_NO_MEMORY:
-            raise MemoryError()
         return None, None
 
     return result, na_count

pandas/_libs/src/parser/tokenizer.c

@@ -28,6 +28,8 @@ GitHub. See Python Software Foundation License and BSD licenses for these.
 #include "pandas/portable.h"
 #include "pandas/vendored/klib/khash.h" // for kh_int64_t, kh_destroy_int64
 
+static const int PROCESSED_WORD_CAPACITY = 128;
+
 void coliter_setup(coliter_t *self, parser_t *parser, int64_t i,
                    int64_t start) {
   // column i, starting at 0
@@ -1874,31 +1876,23 @@ static inline int has_only_spaces(const char *str) {
   return *str == '\0';
 }
 
-/* Copy a string without `char_to_remove`.
- * The returned memory should be free-d with a call to `free`.
+/* Copy a string without `char_to_remove` into `output`,
+ * while ensuring it's null terminated.
  */
-static char *copy_string_without_char(const char *str, char char_to_remove) {
-  size_t chars_to_copy = 0;
-  for (const char *src = str; *src != '\0'; src++) {
+static void copy_string_without_char(char *output, const char *str,
+                                     char char_to_remove, size_t output_size) {
+  size_t i = 0;
+  for (const char *src = str; *src != '\0' && i < output_size; src++) {
     if (*src != char_to_remove) {
-      chars_to_copy++;
+      output[i++] = *src;
     }
   }
-
-  char *start = malloc((chars_to_copy + 1) * sizeof(char));
-  if (!start) {
-    return NULL;
-  }
-
-  char *dst = start;
-  for (const char *src = str; *src != '\0'; src++) {
-    if (*src != char_to_remove) {
-      *dst++ = *src;
-    }
+  if (i < output_size) {
+    output[i] = '\0';
+  } else {
+    // str is too big, probably would overflow
+    errno = ERANGE;
   }
-  *dst = '\0';
-
-  return start;
 }
 
 int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
@@ -1917,19 +1911,19 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
     return 0;
   }
 
-  char *processed_str = NULL;
-
+  errno = 0;
   if (tsep != '\0' && strchr(p_item, tsep) != NULL) {
-    processed_str = copy_string_without_char(p_item, tsep);
-    if (!processed_str) {
-      *error = ERROR_NO_MEMORY;
-      return 0;
-    }
-    p_item = processed_str;
+    char buffer[PROCESSED_WORD_CAPACITY];
+    copy_string_without_char(buffer, p_item, tsep, PROCESSED_WORD_CAPACITY);
+    p_item = buffer;
+  }
+
+  if (errno == ERANGE) {
+    *error = ERROR_OVERFLOW;
+    return 0;
   }
 
   char *endptr = NULL;
-  errno = 0;
   int64_t result = strtoll(p_item, &endptr, 10);
 
   if (!has_only_spaces(endptr)) {
@@ -1944,11 +1938,6 @@ int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
     *error = 0;
   }
 
-  // free processed_str that
-  // was either allocated due to the presence of tsep
-  // or is NULL
-  free(processed_str);
-
   return result;
 }
 
@@ -1977,18 +1966,18 @@ uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
     return 0;
   }
 
-  char *processed_str = NULL;
-
+  errno = 0;
   if (tsep != '\0' && strchr(p_item, tsep) != NULL) {
-    processed_str = copy_string_without_char(p_item, tsep);
-    if (!processed_str) {
-      *error = ERROR_NO_MEMORY;
-      return 0;
-    }
-    p_item = processed_str;
+    char buffer[PROCESSED_WORD_CAPACITY];
+    copy_string_without_char(buffer, p_item, tsep, PROCESSED_WORD_CAPACITY);
+    p_item = buffer;
+  }
+
+  if (errno == ERANGE) {
+    *error = ERROR_OVERFLOW;
+    return 0;
   }
 
-  errno = 0;
   char *endptr = NULL;
   uint64_t result = strtoull(p_item, &endptr, 10);
 
@@ -2006,7 +1995,5 @@ uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
     state->seen_uint = 1;
   }
 
-  free(processed_str);
-
   return result;
 }