release 3.0.2.1

Author: ShenaMao

core/include/ConstantImp.h

@@ -3,10 +3,8 @@
 #include "DolphinDB.h"
 #include "Concurrent.h"
 #include "MultithreadedTableWriter.h"
+
 #include "pybind11/numpy.h"
-#include "pybind11/embed.h"
-#include "pybind11/stl.h"
-#include "Wrappers.h"
 
 #ifdef _MSC_VER
 	#ifdef _USRDLL	
@@ -20,202 +18,9 @@
 
 namespace dolphindb {
 
-
-
-#define EXPARAM_DEFAULT INT_MIN
-
-
-struct HIDEVISIBILITY Preserved {
-    // instantiation only once for frequently use
-    bool np_above_1_20_;
-    bool pd_above_2_0_;
-    bool pd_above_1_2_;
-    bool pyarrow_import_;
-    bool has_arrow_;
-    // modules and methods
-    py::object numpy_;         // module
-    //static py::object isnan_;         // func
-    //static py::object sum_;           // func
-    py::object datetime64_;    // type, equal to np.datetime64
-    py::object nan_;
-    py::object pandas_;        // module
-    py::object pyarrow_;        // pyarrow
-    
-    // pandas types (use py::isinstance)
-    py::object pdseries_;
-    py::object pddataframe_;
-    py::object pdNaT_;
-    py::object pdNA_;//np.nan=pd.NA
-    py::object pdtimestamp_;
-    py::object pdindex_;
-    py::object pdextensiondtype_;
-
-    // pandas extension dtypes (use equal)
-    py::object pdBooleanDtype_;
-    py::object pdFloat32Dtype_;
-    py::object pdFloat64Dtype_;
-    py::object pdInt8Dtype_;
-    py::object pdInt16Dtype_;
-    py::object pdInt32Dtype_;
-    py::object pdInt64Dtype_;
-    py::object pdStringDtype_;
-
-    // numpy dtypes (instances of dtypes, use equal)
-    py::object nparray_;
-    py::object npmatrix_;
-    py::object npbool_;
-    py::object npint8_;
-    py::object npint16_;
-    py::object npint32_;
-    py::object npint64_;
-    py::object npfloat32_;
-    py::object npfloat64_;
-    //npstr has many format, like U4,U8..., they have same type
-    py::object npstrType_;
-
-    py::object npdatetime64M_ins_;
-    py::object npdatetime64D_ins_;
-    py::object npdatetime64m_ins_;
-    py::object npdatetime64s_ins_;
-    py::object npdatetime64h_ins_;
-    py::object npdatetime64ms_ins_;
-    py::object npdatetime64us_ins_;
-    py::object npdatetime64ns_ins_;
-    py::object npdatetime64_ins_;
-
-    py::object npobject_;
-    
-    py::object pynone_;
-    py::object pybool_;
-    py::object pyint_;
-    py::object pyfloat_;
-    py::object pystr_;
-    py::object pybytes_;
-    py::object pyset_;
-    py::object pytuple_;
-    py::object pylist_;
-    py::object pydict_;
-
-    py::object m_decimal_;
-    py::object decimal_;
-
-    // pyarrow types
-    py::object pdarrowdtype_;
-    py::object paboolean_;
-    py::object paint8_;
-    py::object paint16_;
-    py::object paint32_;
-    py::object paint64_;
-    py::object padate32_;
-    py::object patime32_ms_;
-    py::object patime32_s_;
-    py::object patime64_ns_;
-    py::object patimestamp_ns_;
-    py::object patimestamp_ms_;
-    py::object patimestamp_s_;
-    py::object pafloat32_;
-    py::object pafloat64_;
-    py::object padictionary_int32_utf8_;
-    py::object paDictionaryType_;
-    py::object pautf8_;
-    py::object pafixed_size_binary_16_;
-    py::object palarge_binary_;
-    py::object padecimal128_;
-    py::object palist_;
-    py::object pachunkedarray_;
-
-    py::object npdatetime64M_(){ 
-        if(np_above_1_20_) return npdatetime64M_ins_;
-        else return name2dtype("datetime64[M]");
-    }
-    py::object npdatetime64D_(){ 
-        if(np_above_1_20_) return npdatetime64D_ins_;
-        else return name2dtype("datetime64[D]");
-    }
-    py::object npdatetime64m_(){ 
-        if(np_above_1_20_) return npdatetime64m_ins_;
-        else return name2dtype("datetime64[m]");
-    }
-    py::object npdatetime64s_(){
-        if(np_above_1_20_) return npdatetime64s_ins_;
-        else return name2dtype("datetime64[s]");
-    }
-    py::object npdatetime64h_(){ 
-        if(np_above_1_20_) return npdatetime64h_ins_;
-        else return name2dtype("datetime64[h]");
-    }
-    py::object npdatetime64ms_(){ 
-        if(np_above_1_20_) return npdatetime64ms_ins_;
-        else return name2dtype("datetime64[ms]");
-    }
-    py::object npdatetime64us_(){ 
-        if(np_above_1_20_) return npdatetime64us_ins_;
-        else return name2dtype("datetime64[us]");
-    }
-    py::object npdatetime64ns_(){ 
-        if(np_above_1_20_) return npdatetime64ns_ins_;
-        else return name2dtype("datetime64[ns]");
-    }
-    py::object npdatetime64_(){ 
-        if(np_above_1_20_) return npdatetime64_ins_;
-        else return name2dtype("datetime64");
-    }
-
-    Preserved();
-
-    py::object name2dtype(const char *pname){
-        py::object type = py::reinterpret_borrow<py::object>(numpy_.attr("dtype")(pname));
-        // py::dtype type(pname);
-        //py::object dtname=py::getattr(type, "name");
-        //std::string name=py::str(dtname);
-        //printf("DType: %s.",name.data());
-        return type;
-    }
-    static py::object getType(py::object obj){
-        //std::string text=py::str(obj.get_type());
-        //printf("Type: %s.",text.data());
-        return py::reinterpret_borrow<py::object>(obj.get_type());
-    }
-};
-
-enum CHILD_VECTOR_OPTION {
-    DISABLE,
-    ANY_VECTOR,
-    ARRAY_VECTOR,
-};
-
-struct TableVectorInfo {
-    bool        is_empty;
-    std::string colName;
-};
-
-class EXPORT_DECL TableChecker: public std::map<std::string, Type>{
-public:
-    TableChecker() {}
-    TableChecker(const py::object &pydict);
-};
-
 class EXPORT_DECL DdbPythonUtil{
 public:
-    struct ToPythonOption;
-    struct ToPythonOption{
-        bool table2List;//if object is table, false: convert to pandas, true: convert to list
-        ToPythonOption(){
-            table2List = false;
-        }
-        ToPythonOption(bool table2Lista){
-            table2List = table2Lista;
-        }
-    };
-
-    static ConstantSP toDolphinDB(py::object obj, Type typeIndicator = {DT_UNK, EXPARAM_DEFAULT}, TableChecker checker = TableChecker());
-    static py::object toPython(ConstantSP obj, bool tableFlag=false, ToPythonOption *poption = NULL);
-    static ConstantSP toDolphinDB_Scalar(py::object obj, Type typeIndicator = {DT_UNK, EXPARAM_DEFAULT});
-    static ConstantSP toDolphinDB_Vector(py::object obj, Type typeIndicator = {DT_UNK, EXPARAM_DEFAULT}, CHILD_VECTOR_OPTION option = CHILD_VECTOR_OPTION::DISABLE);
     static py::object loadPickleFile(const std::string &filepath);
-    static void createPyVector(const ConstantSP &obj,py::object &pyObject,bool tableFlag,ToPythonOption *poption);
-    static Preserved *preserved_;
-	static void preservedinit() { if(preserved_ == nullptr) preserved_ = new Preserved(); }
 
 protected:
     friend class PytoDdbRowPool;

core/include/DdbPythonUtil.h

@@ -0,0 +1,67 @@
+#ifndef DECIMAL_HELPER_H_
+#define DECIMAL_HELPER_H_
+
+
+#include "ConstantImp.h"
+#include "TypeDefine.h"
+#include "Types.h"
+#include "Util.h"
+
+
+namespace _decimal_util {
+
+using namespace dolphindb;
+using pybind_dolphindb::int128;
+using decimal_util::exp10_i128;
+
+
+inline bool isDecimalType(DATA_TYPE type) {
+    if (type >= ARRAY_TYPE_BASE) {
+        type = static_cast<DATA_TYPE>(type - ARRAY_TYPE_BASE);
+    }
+    return (Util::getCategory(type) == DENARY);
+}
+
+inline int packDecimalTypeAndScale(DATA_TYPE type, int scale) {
+    if (false == isDecimalType(type)) {
+        return static_cast<int>(type);
+    }
+    /*
+     *     31          16            0
+     * +---+-----------+-------------+
+     * | 1 |   scale   |  data type  |
+     * +---+-----------+-------------+
+     */
+    return ((scale << 16) | 0x80000000) | (type & 0xffff);
+}
+
+inline std::pair<DATA_TYPE, int> unpackDecimalTypeAndScale(const int value) {
+    DATA_TYPE type = static_cast<DATA_TYPE>(value);
+    int scale = 0;
+    if (value & 0x80000000) {
+        /*
+         *     31          16            0
+         * +---+-----------+-------------+
+         * | 1 |   scale   |  data type  |
+         * +---+-----------+-------------+
+         */
+        scale = (value & (~0x80000000)) >> 16;
+        type = static_cast<DATA_TYPE>(value & 0xffff);
+        if (type >= ARRAY_TYPE_BASE) {
+            if (Util::getCategory(static_cast<DATA_TYPE>(static_cast<int>(type) - ARRAY_TYPE_BASE)) != DENARY) {
+                type = DT_VOID;
+            }
+        } else {
+            if (Util::getCategory(type) != DENARY) {
+                type = DT_VOID;
+            }
+        }
+    }
+    return {type, scale};
+}
+
+
+} // namespace _decimal_util
+
+
+#endif  // DECIMAL_HELPER_H_

core/include/DecimalHelper.h

@@ -31,13 +31,12 @@
 #include "Table.h"
 #include "pybind11/pybind11.h"
 
+#include "TypeHelper.h"
+
 
 namespace py = pybind11;
 
-#if defined(_MSC_VER)
-#include <BaseTsd.h>
-typedef SSIZE_T ssize_t;
-#endif
+
 
 #ifdef _MSC_VER
 	#if defined(_Py_static_string_init) && defined(PY_MINOR_VERSION) && (PY_MINOR_VERSION >= 10)
@@ -68,7 +67,10 @@ typedef SmartPointer<DBConnection> DBConnectionSP;
 typedef SmartPointer<DBConnectionPool> DBConnectionPoolSP;
 typedef SmartPointer<PartitionedTableAppender> PartitionedTableAppenderSP;
 
-typedef std::pair<DATA_TYPE, int> Type;
+using converter::PyObjs;
+using converter::Type;
+using converter::TableChecker;
+using converter::CHILD_VECTOR_OPTION;
 
 class HIDEVISIBILITY ProtectGil{
 public:

core/include/DolphinDB.h

@@ -39,7 +39,7 @@ class DBConnectionImpl {
     void setProtocol(PROTOCOL protocol) {
         protocol_ = protocol;
         if (protocol_ == PROTOCOL_ARROW) {
-            if (!DdbPythonUtil::preserved_->has_arrow_) {
+            if (!converter::PyObjs::cache_->has_arrow_) {
                 throw RuntimeException("No module named 'pyarrow'");
             }
         }

core/include/DolphinDBImp.h

@@ -0,0 +1,21 @@
+#ifndef IOBINDING_H_
+#define IOBINDING_H_
+
+
+#include <pybind11/pybind11.h>
+#include <pybind11/pytypes.h>
+
+
+namespace py = pybind11;
+
+
+namespace pybind_dolphindb {
+
+void Init_Module_IO(py::module &m);
+
+} // namespace pybind_dolphindb
+
+
+
+
+#endif // IOBINDING_H_

core/include/IOBinding.h

@@ -11,20 +11,24 @@ class EXPORT_DECL Matrix{
     void setRowLabel(const ConstantSP& label);
     void setColumnLabel(const ConstantSP& label);
     bool reshape(INDEX cols, INDEX rows);
-    string getString() const;
-    string getString(INDEX index) const ;
+    std::string getString() const;
+    std::string getString(INDEX index) const ;
     ConstantSP get(const ConstantSP& index) const ;
     bool set(const ConstantSP index, const ConstantSP& value);
-    virtual string getString(int column, int row) const = 0;
+    virtual std::string getString(int column, int row) const = 0;
     virtual ConstantSP getInstance(INDEX size) const = 0;
     virtual ConstantSP getColumn(INDEX index) const = 0;
     virtual bool setColumn(INDEX index, const ConstantSP& value)=0;
     virtual int asof(const ConstantSP& value) const {throw RuntimeException("asof not supported.");}
+
+protected:
+    void calculateInvalidLength(INDEX colStart, int colLength,INDEX rowStart, int rowLength, int& invalidLenBeginning, int& invalidLenEnding) const;
+
 protected:
     int cols_;
     int rows_;
     ConstantSP rowLabel_;
     ConstantSP colLabel_;
 };
 
-}
+}

core/include/Matrix.h

@@ -94,6 +94,7 @@ class EXPORT_DECL MultithreadedTableWriter {
 
 	bool isExit(){ return hasError_; }
     const DATA_TYPE* getColType(){ return colTypes_.data(); }
+    const int* getColExtra(){ return colExtras_.data(); }
     int getColSize(){ return colTypes_.size(); }
 private:
 	bool insert(std::vector<ConstantSP> **records, int recordCount, ErrorCodeInfo &errorInfo);