Support COPY for CSV files (#1371)

* Add more information to CopyStatement. Cleaned up includes.

* Add CSVScan node to ToString and FromString. Removed BINARY external format for now.

* Move COPY from DDL to DML processing. COPY now goes through planner/optimization.

* Propagate external file information

* Removed unused serialization stuff from plan nodes

* Codegen can now have constant generic/opaque bytes in module

* When no columns specified during copy, all columns are inserted

* Added function to throw expception with ill-formatted input string when converting to number

* Removed serialization

* Added input functions in prepartion to read table data from files

* All SQL types must now provide an input function to convert a string into a SQL type

* Added test for value integrity

* First take at CSV Scan translator

* Fix after rebase

* file api

* CSV scanner reads lines

* Process CSV line in scanner

* Free memory when re-allocating line buffer

* Added memcmp to codegen interface. Renamed CallPrintf() to Printf().

* Cleaned up CSV scan translator. Added null checking.

* Moved TupleRuntime::CreateVarlen() into ValuesRuntime::WriteVarlen(). Better code organization and clearer name.

* Added error handling for long columns. Added null-terminator byte for when read-buffers are copied to line-buffers.

* Added inputs for decimal types

* Moved type-specific functions into function namespace

* REALLY simple Date support

* Compile fixes for GCC 6+

* Get string inputs working

* Beefed up tests

* Simple CSV scan test

* Updated optimize to continue support for old/weird/strange AF copy executor

* Extracted implementation into CPP file for plan node

* * Propagatge file options through optimization.
* Added codegen.cpp to source validator whitelist, since we have the ability to call printf() from codegen for debug.
* Beefed up overflow checks in NumericRuntime.
* Fixed tests.

* Fixes after rebase

* Simple function to convert tuple to string CSV

* Fix void* -> i8* conversion

* More tests

* Address reviews

* Revert "Removed serialization"

This reverts commit d055ff9.

* Revert "Removed unused serialization stuff from plan nodes"

This reverts commit 74427c7.

* Beefed up tests, which caught more bugs

* Fix tests

* Reducing copying overhead for columns, constraints and loop variables during CheckConstraints(). We were spending 50% of our time here during bulk insertions into wide tables due to unnecessary copying!

Author: pmenon

script/validators/source_validator.py

@@ -58,12 +58,12 @@
     "src/network/protocol.cpp",
     "src/include/common/macros.h",
     "src/common/stack_trace.cpp",
-    "src/include/parser/sql_scanner.h", # There is a free() in comments
     "src/include/index/bloom_filter.h",
     "src/include/index/compact_ints_key.h",
     "src/include/index/bwtree.h",
     "src/codegen/util/oa_hash_table.cpp",
-    "src/codegen/util/cc_hash_table.cpp"
+    "src/codegen/util/cc_hash_table.cpp",
+    "src/codegen/codegen.cpp",              # We allow calling printf() from codegen for debugging
 ]
 
 ## ==============================================

src/binder/bind_node_visitor.cpp

@@ -166,7 +166,19 @@ void BindNodeVisitor::Visit(parser::DeleteStatement *node) {
 }
 
 void BindNodeVisitor::Visit(parser::LimitDescription *) {}
-void BindNodeVisitor::Visit(parser::CopyStatement *) {}
+
+void BindNodeVisitor::Visit(parser::CopyStatement *node) {
+  context_ = std::make_shared<BinderContext>(nullptr);
+  if (node->table != nullptr) {
+    node->table->Accept(this);
+
+    // If the table is given, we're either writing or reading all columns
+    context_->GenerateAllColumnExpressions(node->select_list);
+  } else {
+    node->select_stmt->Accept(this);
+  }
+}
+
 void BindNodeVisitor::Visit(parser::CreateFunctionStatement *) {}
 void BindNodeVisitor::Visit(parser::CreateStatement *node) {
   node->TryBindDatabaseName(default_database_name_);

src/catalog/abstract_catalog.cpp

@@ -35,6 +35,7 @@
 #include "executor/plan_executor.h"
 #include "executor/seq_scan_executor.h"
 #include "executor/update_executor.h"
+#include "expression/constant_value_expression.h"
 
 #include "storage/database.h"
 #include "storage/storage_manager.h"

src/catalog/catalog.cpp

@@ -30,7 +30,7 @@
 #include "codegen/code_context.h"
 #include "concurrency/transaction_manager_factory.h"
 #include "function/date_functions.h"
-#include "function/decimal_functions.h"
+#include "function/numeric_functions.h"
 #include "function/old_engine_string_functions.h"
 #include "function/timestamp_functions.h"
 #include "index/index_factory.h"
@@ -1283,43 +1283,43 @@ void Catalog::InitializeFunctions() {
       AddBuiltinFunction("abs", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL,
                          internal_lang, "Abs",
                          function::BuiltInFuncType{
-                             OperatorId::Abs, function::DecimalFunctions::_Abs},
+                             OperatorId::Abs, function::NumericFunctions::_Abs},
                          txn);
       AddBuiltinFunction(
           "sqrt", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, internal_lang,
           "Sqrt",
           function::BuiltInFuncType{OperatorId::Sqrt,
-                                    function::DecimalFunctions::Sqrt},
+                                    function::NumericFunctions::Sqrt},
           txn);
       AddBuiltinFunction(
           "sqrt", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL,
           internal_lang, "Sqrt",
           function::BuiltInFuncType{OperatorId::Sqrt,
-                                    function::DecimalFunctions::Sqrt},
+                                    function::NumericFunctions::Sqrt},
           txn);
       AddBuiltinFunction(
           "sqrt", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, internal_lang,
           "Sqrt",
           function::BuiltInFuncType{OperatorId::Sqrt,
-                                    function::DecimalFunctions::Sqrt},
+                                    function::NumericFunctions::Sqrt},
           txn);
       AddBuiltinFunction(
           "sqrt", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang,
           "Sqrt",
           function::BuiltInFuncType{OperatorId::Sqrt,
-                                    function::DecimalFunctions::Sqrt},
+                                    function::NumericFunctions::Sqrt},
           txn);
       AddBuiltinFunction(
           "sqrt", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, internal_lang,
           "Sqrt",
           function::BuiltInFuncType{OperatorId::Sqrt,
-                                    function::DecimalFunctions::Sqrt},
+                                    function::NumericFunctions::Sqrt},
           txn);
       AddBuiltinFunction(
           "floor", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL,
           internal_lang, "Floor",
           function::BuiltInFuncType{OperatorId::Floor,
-                                    function::DecimalFunctions::_Floor},
+                                    function::NumericFunctions::_Floor},
           txn);
 
       /**
@@ -1328,126 +1328,126 @@ void Catalog::InitializeFunctions() {
       AddBuiltinFunction("abs", {type::TypeId::TINYINT}, type::TypeId::TINYINT,
                          internal_lang, "Abs",
                          function::BuiltInFuncType{
-                             OperatorId::Abs, function::DecimalFunctions::_Abs},
+                             OperatorId::Abs, function::NumericFunctions::_Abs},
                          txn);
 
       AddBuiltinFunction("abs", {type::TypeId::SMALLINT},
                          type::TypeId::SMALLINT, internal_lang, "Abs",
                          function::BuiltInFuncType{
-                             OperatorId::Abs, function::DecimalFunctions::_Abs},
+                             OperatorId::Abs, function::NumericFunctions::_Abs},
                          txn);
 
       AddBuiltinFunction("abs", {type::TypeId::INTEGER}, type::TypeId::INTEGER,
                          internal_lang, "Abs",
                          function::BuiltInFuncType{
-                             OperatorId::Abs, function::DecimalFunctions::_Abs},
+                             OperatorId::Abs, function::NumericFunctions::_Abs},
                          txn);
 
       AddBuiltinFunction("abs", {type::TypeId::BIGINT}, type::TypeId::BIGINT,
                          internal_lang, "Abs",
                          function::BuiltInFuncType{
-                             OperatorId::Abs, function::DecimalFunctions::_Abs},
+                             OperatorId::Abs, function::NumericFunctions::_Abs},
                          txn);
 
       AddBuiltinFunction(
           "floor", {type::TypeId::INTEGER}, type::TypeId::DECIMAL,
           internal_lang, "Floor",
           function::BuiltInFuncType{OperatorId::Floor,
-                                    function::DecimalFunctions::_Floor},
+                                    function::NumericFunctions::_Floor},
           txn);
       AddBuiltinFunction(
           "floor", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang,
           "Floor",
           function::BuiltInFuncType{OperatorId::Floor,
-                                    function::DecimalFunctions::_Floor},
+                                    function::NumericFunctions::_Floor},
           txn);
       AddBuiltinFunction(
           "floor", {type::TypeId::TINYINT}, type::TypeId::DECIMAL,
           internal_lang, "Floor",
           function::BuiltInFuncType{OperatorId::Floor,
-                                    function::DecimalFunctions::_Floor},
+                                    function::NumericFunctions::_Floor},
           txn);
       AddBuiltinFunction(
           "floor", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL,
           internal_lang, "Floor",
           function::BuiltInFuncType{OperatorId::Floor,
-                                    function::DecimalFunctions::_Floor},
+                                    function::NumericFunctions::_Floor},
           txn);
       AddBuiltinFunction(
           "round", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL,
           internal_lang, "Round",
           function::BuiltInFuncType{OperatorId::Round,
-                                    function::DecimalFunctions::_Round},
+                                    function::NumericFunctions::_Round},
           txn);
 
       AddBuiltinFunction(
           "ceil", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL, internal_lang,
           "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceil", {type::TypeId::TINYINT}, type::TypeId::DECIMAL, internal_lang,
           "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceil", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceil", {type::TypeId::INTEGER}, type::TypeId::DECIMAL, internal_lang,
           "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceil", {type::TypeId::BIGINT}, type::TypeId::DECIMAL, internal_lang,
           "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceiling", {type::TypeId::DECIMAL}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceiling", {type::TypeId::TINYINT}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceiling", {type::TypeId::SMALLINT}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceiling", {type::TypeId::INTEGER}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       AddBuiltinFunction(
           "ceiling", {type::TypeId::BIGINT}, type::TypeId::DECIMAL,
           internal_lang, "Ceil",
           function::BuiltInFuncType{OperatorId::Ceil,
-                                    function::DecimalFunctions::_Ceil},
+                                    function::NumericFunctions::_Ceil},
           txn);
 
       /**

src/codegen/buffering_consumer.cpp

@@ -40,6 +40,15 @@ WrappedTuple &WrappedTuple::operator=(const WrappedTuple &o) {
   return *this;
 }
 
+std::string WrappedTuple::ToCSV() const {
+  std::string ret;
+  for (uint32_t i = 0; i < tuple_.size(); i++) {
+    if (i != 0) ret.append(",");
+    ret.append(tuple_[i].ToString());
+  }
+  return ret;
+}
+
 //===----------------------------------------------------------------------===//
 // BufferTuple() Proxy
 //===----------------------------------------------------------------------===//

src/codegen/codegen.cpp

@@ -59,12 +59,30 @@ llvm::Constant *CodeGen::ConstDouble(double val) const {
   return llvm::ConstantFP::get(DoubleType(), val);
 }
 
-llvm::Constant *CodeGen::ConstString(const std::string &s) const {
+llvm::Value *CodeGen::ConstString(const std::string &str_val,
+                                  const std::string &name) const {
   // Strings are treated as arrays of bytes
-  auto *str = llvm::ConstantDataArray::getString(GetContext(), s);
-  return new llvm::GlobalVariable(GetModule(), str->getType(), true,
-                                  llvm::GlobalValue::InternalLinkage, str,
-                                  "str");
+  auto *str = llvm::ConstantDataArray::getString(GetContext(), str_val);
+  auto *global_var =
+      new llvm::GlobalVariable(GetModule(), str->getType(), true,
+                               llvm::GlobalValue::InternalLinkage, str, name);
+  return GetBuilder().CreateInBoundsGEP(global_var, {Const32(0), Const32(0)});
+}
+
+llvm::Value *CodeGen::ConstGenericBytes(const void *data, uint32_t length,
+                                        const std::string &name) const {
+  // Create the constant data array that wraps the input data
+  llvm::ArrayRef<uint8_t> elements{reinterpret_cast<const uint8_t *>(data),
+                                   length};
+  auto *arr = llvm::ConstantDataArray::get(GetContext(), elements);
+
+  // Create a global variable for the data
+  auto *global_var =
+      new llvm::GlobalVariable(GetModule(), arr->getType(), true,
+                               llvm::GlobalValue::InternalLinkage, arr, name);
+
+  // Return a pointer to the first element
+  return GetBuilder().CreateInBoundsGEP(global_var, {Const32(0), Const32(0)});
 }
 
 llvm::Constant *CodeGen::Null(llvm::Type *type) const {
@@ -75,11 +93,6 @@ llvm::Constant *CodeGen::NullPtr(llvm::PointerType *type) const {
   return llvm::ConstantPointerNull::get(type);
 }
 
-llvm::Value *CodeGen::ConstStringPtr(const std::string &s) const {
-  auto &ir_builder = GetBuilder();
-  return ir_builder.CreateConstInBoundsGEP2_32(nullptr, ConstString(s), 0, 0);
-}
-
 llvm::Value *CodeGen::AllocateVariable(llvm::Type *type,
                                        const std::string &name) {
   // To allocate a variable, a function must be under construction
@@ -135,26 +148,68 @@ llvm::Value *CodeGen::CallFunc(llvm::Value *fn,
   return GetBuilder().CreateCall(fn, args);
 }
 
-llvm::Value *CodeGen::CallPrintf(const std::string &format,
-                                 const std::vector<llvm::Value *> &args) {
+llvm::Value *CodeGen::Printf(const std::string &format,
+                             const std::vector<llvm::Value *> &args) {
   auto *printf_fn = LookupBuiltin("printf");
   if (printf_fn == nullptr) {
+#if GCC_AT_LEAST_6
+// In newer GCC versions (i.e., GCC 6+), function attributes are part of the
+// type system and are attached to the function signature. For example, printf()
+// comes with the "noexcept" attribute. Moreover, GCC 6+ will complain when
+// attributes attached to a function (e.g., noexcept()) are not used at
+// their call-site. Below, we use decltype(printf) to get the C/C++ function
+// type of printf(...), but we discard the attributes since we don't need
+// them. Hence, on GCC 6+, compilation will fail without adding the
+// "-Wignored-attributes" flag. So, we add it here only.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
     printf_fn = RegisterBuiltin(
-        "printf", llvm::TypeBuilder<int(char *, ...), false>::get(GetContext()),
+        "printf", llvm::TypeBuilder<decltype(printf), false>::get(GetContext()),
         reinterpret_cast<void *>(printf));
+#if GCC_AT_LEAST_6
+#pragma GCC diagnostic pop
+#endif
   }
-  auto &ir_builder = code_context_.GetBuilder();
-  auto *format_str =
-      ir_builder.CreateGEP(ConstString(format), {Const32(0), Const32(0)});
 
   // Collect all the arguments into a vector
-  std::vector<llvm::Value *> printf_args{format_str};
+  std::vector<llvm::Value *> printf_args = {ConstString(format, "format")};
   printf_args.insert(printf_args.end(), args.begin(), args.end());
 
-  // Call the function
+  // Call printf()
   return CallFunc(printf_fn, printf_args);
 }
 
+llvm::Value *CodeGen::Memcmp(llvm::Value *ptr1, llvm::Value *ptr2,
+                             llvm::Value *len) {
+  static constexpr char kMemcmpFnName[] = "memcmp";
+  auto *memcmp_fn = LookupBuiltin(kMemcmpFnName);
+  if (memcmp_fn == nullptr) {
+#if GCC_AT_LEAST_6
+// In newer GCC versions (i.e., GCC 6+), function attributes are part of the
+// type system and are attached to the function signature. For example, memcmp()
+// comes with the "throw()" attribute, among many others. Moreover, GCC 6+ will
+// complain when attributes attached to a function are not used at their
+// call-site. Below, we use decltype(memcmp) to get the C/C++ function type
+// of memcmp(...), but we discard the attributes since we don't need them.
+// Hence, on GCC 6+, compilation will fail without adding the
+// "-Wignored-attributes" flag. So, we add it here only.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+    memcmp_fn = RegisterBuiltin(
+        kMemcmpFnName,
+        llvm::TypeBuilder<decltype(memcmp), false>::get(GetContext()),
+        reinterpret_cast<void *>(memcmp));
+#if GCC_AT_LEAST_6
+#pragma GCC diagnostic pop
+#endif
+  }
+
+  // Call memcmp()
+  return CallFunc(memcmp_fn, {ptr1, ptr2, len});
+}
+
 llvm::Value *CodeGen::Sqrt(llvm::Value *val) {
   llvm::Function *sqrt_func = llvm::Intrinsic::getDeclaration(
       &GetModule(), llvm::Intrinsic::sqrt, val->getType());