[chore] add "copy from file" grammay.

Signed-off-by: clundro <[email protected]>

Author: infdahai

build_support/generate_flex.py

@@ -0,0 +1,84 @@
+# use flex to generate the scanner file for the parser
+# the following version of bison is used:
+# flex 2.5.35 Apple(flex-32)
+import os
+import subprocess
+import re
+from sys import platform
+import sys
+
+def open_utf8(fpath, flags):
+    import sys
+    if sys.version_info[0] < 3:
+        return open(fpath, flags)
+    else:
+        return open(fpath, flags, encoding="utf8")
+
+flex_bin = 'flex'
+for arg in sys.argv[1:]:
+    if arg.startswith("--flex="):
+        flex_bin = arg.replace("--flex=", "")
+
+pg_path = os.path.join('third_party', 'libpg_query')
+flex_file_path = os.path.join(pg_path, 'scan.l')
+target_file = os.path.join(pg_path, 'src_backend_parser_scan.cpp')
+
+proc = subprocess.Popen([flex_bin, '--nounistd', '-o', target_file,
+                        flex_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+stdout = proc.stdout.read().decode('utf8')
+stderr = proc.stderr.read().decode('utf8')
+if proc.returncode != None or len(stderr) > 0:
+    print("Flex failed")
+    print("stdout: ", stdout)
+    print("stderr: ", stderr)
+    exit(1)
+
+with open_utf8(target_file, 'r') as f:
+    text = f.read()
+
+# convert this from 'int' to 'yy_size_t' to avoid triggering a warning
+text = text.replace('int yy_buf_size;\n', 'yy_size_t yy_buf_size;\n')
+
+# add the libpg_query namespace
+text = text.replace('''
+#ifndef FLEXINT_H
+#define FLEXINT_H
+''', '''
+#ifndef FLEXINT_H
+#define FLEXINT_H
+namespace duckdb_libpgquery {
+''')
+text = text.replace('register ', '')
+
+text = text + "\n} /* duckdb_libpgquery */\n"
+
+text = re.sub('(?:[(]void[)][ ]*)?fprintf', '//', text)
+text = re.sub('exit[(]', 'throw std::runtime_error(msg); //', text)
+text = re.sub(
+    r'\n\s*if\s*[(]\s*!\s*yyin\s*[)]\s*\n\s*yyin\s*=\s*stdin;\s*\n', '\n', text)
+text = re.sub(
+    r'\n\s*if\s*[(]\s*!\s*yyout\s*[)]\s*\n\s*yyout\s*=\s*stdout;\s*\n', '\n', text)
+
+file_null = 'NULL' if platform == 'linux' else '[(]FILE [*][)] 0'
+
+text = re.sub(
+    rf'[#]ifdef\s*YY_STDINIT\n\s*yyin = stdin;\n\s*yyout = stdout;\n[#]else\n\s*yyin = {file_null};\n\s*yyout = {file_null};\n[#]endif', '    yyin = (FILE *) 0;\n    yyout = (FILE *) 0;', text)
+
+if 'stdin;' in text:
+    print("STDIN not removed!")
+    # exit(1)
+
+if 'stdout' in text:
+    print("STDOUT not removed!")
+    # exit(1)
+
+if 'fprintf(' in text:
+    print("PRINTF not removed!")
+    # exit(1)
+
+if 'exit(' in text:
+    print("EXIT not removed!")
+    # exit(1)
+
+with open_utf8(target_file, 'w+') as f:
+    f.write(text)

test/binder/binder_test.cpp

@@ -173,6 +173,11 @@ TEST(BinderTest, BindBinaryOp) {
   PrintStatements(statements);
 }
 
+TEST(BinderTest, BindCopyFrom) {
+  auto statements = TryBind("copy from");
+  PrintStatements(statements);
+}
+
 // TODO(chi): subquery is not supported yet
 TEST(BinderTest, DISABLED_BindUncorrelatedSubquery) {
   auto statements = TryBind("select * from (select * from a) INNER JOIN (select * from b) ON a.x = b.y");

third_party/libpg_query/grammar/statements/copy.y

@@ -1,4 +1,23 @@
-CopyStmt:	COPY opt_binary qualified_name opt_column_list opt_oids
+CopyStmt:	COPY FROM copy_file_name
+				{
+					PGCopyStmt *n = makeNode(PGCopyStmt);
+					n->relation = NULL;
+					n->query = NULL;
+					n->attlist = NIL;
+					n->is_from = true;
+					n->is_program = true;
+					n->filename = $3;
+					n->options = NIL;
+
+					if (n->is_program && n->filename == NULL)
+						ereport(ERROR,
+								(errcode(PG_ERRCODE_SYNTAX_ERROR),
+								 errmsg("COPYFROMFILE not allowed with NULL"),
+								 parser_errposition(@3)));
+
+					$$ = (PGNode *)n;
+				}
+			|	COPY opt_binary qualified_name opt_column_list opt_oids
 			copy_from opt_program copy_file_name copy_delimiter opt_with copy_options
 				{
 					PGCopyStmt *n = makeNode(PGCopyStmt);

third_party/libpg_query/src_backend_parser_scan.cpp

@@ -1,4 +1,4 @@
-#line 1 "third_party/libpg_query/src_backend_parser_scan.cpp"
+#line 2 "third_party/libpg_query/src_backend_parser_scan.cpp"
 /*-------------------------------------------------------------------------
  *
  * scan.l
@@ -40,7 +40,7 @@
 
 #include <stdexcept>
 
-#line 43 "third_party/libpg_query/src_backend_parser_scan.cpp"
+#line 44 "third_party/libpg_query/src_backend_parser_scan.cpp"
 
 #define  YY_INT_ALIGNED short int
 
@@ -322,7 +322,6 @@ typedef int16_t flex_int16_t;
 typedef uint16_t flex_uint16_t;
 typedef int32_t flex_int32_t;
 typedef uint32_t flex_uint32_t;
-typedef uint64_t flex_uint64_t;
 #else
 typedef signed char flex_int8_t;
 typedef short int flex_int16_t;
@@ -482,12 +481,12 @@ struct yy_buffer_state
 	/* Size of input buffer in bytes, not including room for EOB
 	 * characters.
 	 */
-	int yy_buf_size;
+	yy_size_t yy_buf_size;
 
 	/* Number of characters read into yy_ch_buf, not including EOB
 	 * characters.
 	 */
-	yy_size_t yy_n_chars;
+	int yy_n_chars;
 
 	/* Whether we "own" the buffer - i.e., we know we created it,
 	 * and can realloc() it to grow it, and should free() it to
@@ -564,7 +563,7 @@ static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file , yyscan_t yyscanner
 
 YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner );
 YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner );
-YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, yy_size_t len , yyscan_t yyscanner );
+YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner );
 
 void *yyalloc ( yy_size_t , yyscan_t yyscanner );
 void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner );
@@ -611,7 +610,7 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner );
  */
 #define YY_DO_BEFORE_ACTION \
 	yyg->yytext_ptr = yy_bp; \
-	yyleng = (yy_size_t) (yy_cp - yy_bp); \
+	yyleng = (int) (yy_cp - yy_bp); \
 	yyg->yy_hold_char = *yy_cp; \
 	*yy_cp = '\0'; \
 	yyg->yy_c_buf_p = yy_cp;
@@ -1336,8 +1335,8 @@ struct yyguts_t
     size_t yy_buffer_stack_max; /**< capacity of stack. */
     YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */
     char yy_hold_char;
-    yy_size_t yy_n_chars;
-    yy_size_t yyleng_r;
+    int yy_n_chars;
+    int yyleng_r;
     char *yy_c_buf_p;
     int yy_init;
     int yy_start;
@@ -1394,7 +1393,7 @@ FILE *yyget_out ( yyscan_t yyscanner );
 
 void yyset_out  ( FILE * _out_str , yyscan_t yyscanner );
 
-			yy_size_t yyget_leng ( yyscan_t yyscanner );
+			int yyget_leng ( yyscan_t yyscanner );
 
 char *yyget_text ( yyscan_t yyscanner );
 
@@ -1473,7 +1472,7 @@ static int input ( yyscan_t yyscanner );
 	if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
 		{ \
 		int c = '*'; \
-		yy_size_t n; \
+		int n; \
 		for ( n = 0; n < max_size && \
 			     (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
 			buf[n] = (char) c; \
@@ -2839,7 +2838,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 	else
 		{
-			yy_size_t num_to_read =
+			int num_to_read =
 			YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1;
 
 		while ( num_to_read <= 0 )
@@ -2853,7 +2852,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 			if ( b->yy_is_our_buffer )
 				{
-				yy_size_t new_size = b->yy_buf_size * 2;
+				int new_size = b->yy_buf_size * 2;
 
 				if ( new_size <= 0 )
 					b->yy_buf_size += b->yy_buf_size / 8;
@@ -2911,7 +2910,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 	if ((yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) {
 		/* Extend the array by 50%, plus the number we really need. */
-		yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1);
+		int new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1);
 		YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc(
 			(void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size , yyscanner );
 		if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf )
@@ -3018,7 +3017,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 
 		else
 			{ /* need more input */
-			yy_size_t offset = yyg->yy_c_buf_p - yyg->yytext_ptr;
+			int offset = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr);
 			++yyg->yy_c_buf_p;
 
 			switch ( yy_get_next_buffer( yyscanner ) )
@@ -3396,12 +3395,12 @@ YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner)
  * @param yyscanner The scanner object.
  * @return the newly allocated buffer state object.
  */
-YY_BUFFER_STATE yy_scan_bytes  (const char * yybytes, yy_size_t  _yybytes_len , yyscan_t yyscanner)
+YY_BUFFER_STATE yy_scan_bytes  (const char * yybytes, int  _yybytes_len , yyscan_t yyscanner)
 {
 	YY_BUFFER_STATE b;
 	char *buf;
 	yy_size_t n;
-	yy_size_t i;
+	int i;
 
 	/* Get memory for full buffer, including space for trailing EOB's. */
 	n = (yy_size_t) (_yybytes_len + 2);
@@ -3445,7 +3444,7 @@ static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner)
 	do \
 		{ \
 		/* Undo effects of setting up yytext. */ \
-        yy_size_t yyless_macro_arg = (n); \
+        int yyless_macro_arg = (n); \
         YY_LESS_LINENO(yyless_macro_arg);\
 		yytext[yyleng] = yyg->yy_hold_char; \
 		yyg->yy_c_buf_p = yytext + yyless_macro_arg; \
@@ -3513,7 +3512,7 @@ FILE *yyget_out  (yyscan_t yyscanner)
 /** Get the length of the current token.
  * @param yyscanner The scanner object.
  */
-yy_size_t yyget_leng  (yyscan_t yyscanner)
+int yyget_leng  (yyscan_t yyscanner)
 {
     struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
     return yyleng;
@@ -3704,13 +3703,8 @@ static int yy_init_globals (yyscan_t yyscanner)
     yyg->yy_start_stack =  NULL;
 
 /* Defined in main.c */
-#ifdef YY_STDINIT
-    yyin = stdin;
-    yyout = stdout;
-#else
-    yyin = NULL;
-    yyout = NULL;
-#endif
+    yyin = (FILE *) 0;
+    yyout = (FILE *) 0;
 
     /* For future reference: Set errno on error, since we are called by
      * yylex_init()