release c schema and arrow, working version

Author: sfc-gh-aling

MANIFEST.in

@@ -7,7 +7,9 @@ recursive-include src/snowflake/connector py.typed *.py *.pyx
 recursive-include src/snowflake/connector/vendored LICENSE*
 
 recursive-include src/snowflake/connector/cpp *.cpp *.hpp
+recursive-include src/snowflake/connector/cpp *.c *.h
 exclude src/snowflake/connector/arrow_iterator.cpp
+exclude src/snowflake/connector/cpp/ArrowIterator/arrow_iterator.cpp
 
 exclude .git-blame-ignore-revs
 exclude .pre-commit-config.yaml
@@ -19,6 +21,7 @@ prune ci
 prune benchmark
 prune test
 prune tested_requirements
+prune stress
 prune src/snowflake/connector/cpp/scripts
 prune __pycache__
 prune samples

src/snowflake/connector/cpp/ArrowIterator/CArrowTableIterator.cpp

@@ -336,7 +336,9 @@ void CArrowTableIterator::convertScaledFixedNumberColumnToDecimalColumn_nanoarro
     }
   }
   ArrowArrayFinishBuildingDefault(newArray, &error);
+  m_nanoarrowSchemas[batchIdx]->children[colIdx]->release(m_nanoarrowSchemas[batchIdx]->children[colIdx]);
   ArrowSchemaMove(newSchema, m_nanoarrowSchemas[batchIdx]->children[colIdx]);
+  m_nanoarrowTable[batchIdx]->children[colIdx]->release(m_nanoarrowTable[batchIdx]->children[colIdx]);
   ArrowArrayMove(newArray, m_nanoarrowTable[batchIdx]->children[colIdx]);
   m_newArrays[batchIdx].push_back(std::move(newUniqueArray));
   m_newSchemas[batchIdx].push_back(std::move(newUniqueField));
@@ -384,7 +386,9 @@ void CArrowTableIterator::convertScaledFixedNumberColumnToDoubleColumn_nanoarrow
     }
   }
   ArrowArrayFinishBuildingDefault(newArray, &error);
+  m_nanoarrowSchemas[batchIdx]->children[colIdx]->release(m_nanoarrowSchemas[batchIdx]->children[colIdx]);
   ArrowSchemaMove(newSchema, m_nanoarrowSchemas[batchIdx]->children[colIdx]);
+  m_nanoarrowTable[batchIdx]->children[colIdx]->release(m_nanoarrowTable[batchIdx]->children[colIdx]);
   ArrowArrayMove(newArray, m_nanoarrowTable[batchIdx]->children[colIdx]);
   m_newArrays[batchIdx].push_back(std::move(newUniqueArray));
   m_newSchemas[batchIdx].push_back(std::move(newUniqueField));
@@ -457,7 +461,9 @@ void CArrowTableIterator::convertTimeColumn_nanoarrow(
   }
 
   ArrowArrayFinishBuildingDefault(newArray, &error);
+  m_nanoarrowSchemas[batchIdx]->children[colIdx]->release(m_nanoarrowSchemas[batchIdx]->children[colIdx]);
   ArrowSchemaMove(newSchema, m_nanoarrowSchemas[batchIdx]->children[colIdx]);
+  m_nanoarrowTable[batchIdx]->children[colIdx]->release(m_nanoarrowTable[batchIdx]->children[colIdx]);
   ArrowArrayMove(newArray, m_nanoarrowTable[batchIdx]->children[colIdx]);
   m_newArrays[batchIdx].push_back(std::move(newUniqueArray));
   m_newSchemas[batchIdx].push_back(std::move(newUniqueField));
@@ -675,7 +681,9 @@ void CArrowTableIterator::convertTimestampColumn_nanoarrow(
   }
 
   ArrowArrayFinishBuildingDefault(newArray, &error);
+  m_nanoarrowSchemas[batchIdx]->children[colIdx]->release(m_nanoarrowSchemas[batchIdx]->children[colIdx]);
   ArrowSchemaMove(newSchema, m_nanoarrowSchemas[batchIdx]->children[colIdx]);
+  m_nanoarrowTable[batchIdx]->children[colIdx]->release(m_nanoarrowTable[batchIdx]->children[colIdx]);
   ArrowArrayMove(newArray, m_nanoarrowTable[batchIdx]->children[colIdx]);
   m_newArrays[batchIdx].push_back(std::move(newUniqueArray));
   m_newSchemas[batchIdx].push_back(std::move(newUniqueField));
@@ -813,7 +821,9 @@ void CArrowTableIterator::convertTimestampTZColumn_nanoarrow(
   }
 
   ArrowArrayFinishBuildingDefault(newArray, &error);
+  m_nanoarrowSchemas[batchIdx]->children[colIdx]->release(m_nanoarrowSchemas[batchIdx]->children[colIdx]);
   ArrowSchemaMove(newSchema, m_nanoarrowSchemas[batchIdx]->children[colIdx]);
+  m_nanoarrowTable[batchIdx]->children[colIdx]->release(m_nanoarrowTable[batchIdx]->children[colIdx]);
   ArrowArrayMove(newArray, m_nanoarrowTable[batchIdx]->children[colIdx]);
   m_newArrays[batchIdx].push_back(std::move(newUniqueArray));
   m_newSchemas[batchIdx].push_back(std::move(newUniqueField));

stress/unit/local_iterator.py

@@ -32,6 +32,20 @@ def task_for_loop_iterator(bytes_data):
         pass
 
 
+def task_for_loop_table_iterator(bytes_data):
+    arrow_iter = PyArrowIterator(
+        None,
+        io.BytesIO(bytes_data),
+        arrow_context,
+        False,
+        False,
+        False,
+    )
+    arrow_iter.init_table_unit()
+    for _ in arrow_iter:
+        pass
+
+
 def execute_task(task, bytes_data):
     for _ in range(iteration_cnt):
         task(bytes_data)
@@ -40,9 +54,11 @@ def execute_task(task, bytes_data):
 if __name__ == "__main__":
 
     perf_check_task_for_loop_iterator = task_time_execution_decorator(
-        task_for_loop_iterator
+        task_for_loop_table_iterator
+    )
+    memory_check_task_for_loop_iterator = task_memory_decorator(
+        task_for_loop_table_iterator
     )
-    memory_check_task_for_loop_iterator = task_memory_decorator(task_for_loop_iterator)
 
     execute_task(memory_check_task_for_loop_iterator, decoded_bytes)
     memory_records = stress_util.collect_memory_records()