HAWQ-1720. support create table in orc format

apache · May 22, 2019 · b935042 · b935042
1 parent 79869d6
commit b935042
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 25 deletions.
diff --git a/contrib/exthdfs/Makefile b/contrib/exthdfs/Makefile
@@ -21,7 +21,7 @@ OBJS       = exthdfs.o
 PG_CPPFLAGS = -I$(libpq_srcdir)
 PG_LIBS = $(libpq_pgport)
 
-override CFLAGS += -lhdfs3
+override CFLAGS += -lhdfs3 -lstorage
 
 ifdef USE_PGXS
 PGXS := $(shell pg_config --pgxs)

diff --git a/contrib/exthdfs/exthdfs.c b/contrib/exthdfs/exthdfs.c
@@ -22,13 +22,11 @@
 #include "common.h"
 #include "access/extprotocol.h"
 #include "cdb/cdbdatalocality.h"
-#include "storage/fd.h"
-#include "storage/filesystem.h"
+#include "cdb/cdbfilesystemcredential.h"
+#include "cdb/cdbvars.h"
+#include "storage/cwrapper/hdfs-file-system-c.h"
 #include "utils/uri.h"
 
-
-
-
 PG_MODULE_MAGIC;
 
 PG_FUNCTION_INFO_V1(hdfsprotocol_blocklocation);
@@ -421,30 +419,47 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS)
 		}
 
 		/* SHOULD ADD LOGIC HERE TO CREATE UNEXISTING PATH */
-		if (pvalidator_data->forceCreateDir) {
-
+		if (pvalidator_data->forceCreateDir)
+		{
 		  elog(LOG, "hdfs_validator() forced creating dir");
 
-		  /* Create file system instance */
-		  	hdfsFS fs = hdfsConnect(uri->hostname, uri->port);
+			/* Create file system instance */
+			FscHdfsFileSystemC *fs = FscHdfsNewFileSystem(uri->hostname,
+					uri->port);
 			if (fs == NULL)
 			{
 				elog(ERROR, "hdfsprotocol_validate : "
-							"failed to create HDFS instance to connect to %s:%d",
-							uri->hostname, uri->port);
+				"failed to create HDFS instance to connect to %s:%d",
+				uri->hostname, uri->port);
 			}
 
-			if (hdfsExists(fs, uri->path) == -1)
+			if (FscHdfsExistPath(fs, uri->path)
+					&& FscHdfsGetFileKind(fs, uri->path) == 'F')
 				elog(ERROR, "hdfsprotocol_validate : "
-						"Location \"%s\" is not exist",
-						uri->path);
+				"Location \"%s\" is a file, not supported yet. "
+				"Only support directory now",
+				uri->path);
+			if (pvalidator_data->direction == EXT_VALIDATE_WRITE
+					&& !FscHdfsExistInsertPath(fs, uri->path))
+			{
+				elog(LOG, "hdfs_validator() to create url %s", uri->path);
+				FscHdfsCreateInsertDir(fs, uri->path);
+				if (FscHdfsHasErrorRaised(fs))
+				{
+					CatchedError *ce = FscHdfsGetFileSystemError(fs);
+					elog(ERROR, "hdfsprotocol_validate : "
+					"failed to create directory %s : %s(%d)",
+					uri->path,
+					ce->errMessage, ce->errCode);
+				}
+			}
 
-		 /* destroy fs instance */
-			hdfsDisconnect(fs);
+			/* destroy fs instance */
+			FscHdfsFreeFileSystemC(&fs);
 		}
 
 		/* Clean up temporarily created instances */
-		FreeExternalTableUri(uri);
+		pfree(uri);
 		if (nnaddr != NULL)
 		{
 			pfree(nnaddr);

diff --git a/src/backend/catalog/gp_toolkit.sql.in b/src/backend/catalog/gp_toolkit.sql.in
@@ -179,7 +179,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
 --		External table that determines the local segment id
 --		
 --------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid
 -- (
 -- 	localid	int
 -- )
@@ -196,7 +196,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
 --		External table that determines the master's segment id
 --		
 --------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid
 -- (
 -- 	masterid	int
 -- )
@@ -238,7 +238,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
 --		External table to read segment log; requires superuser privilege
 --		
 --------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext
 -- (
 -- 	logtime timestamp with time zone,
 -- 	loguser text,
@@ -285,7 +285,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
 --		External table to read the master log; requires superuser privilege
 --		
 --------------------------------------------------------------------------------
-CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
+CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
 (
 	logtime timestamp with time zone,
 	loguser text,
@@ -319,7 +319,7 @@ CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
 	logstack text
 )
 EXECUTE E'cat $GP_SEG_DATADIR/pg_log/*.csv' ON MASTER
-FORMAT 'CSV' (DELIMITER AS ',' NULL AS '' QUOTE AS '"');
+FORMAT 'CSV' (DELIMITER ',' NULL '' QUOTE '"');
 
 REVOKE ALL ON TABLE %%JETPACK_PREFIX_AUX%%log_master_ext FROM public;
 
@@ -994,7 +994,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX%%stats_missing TO public;
 --		External table to determine free space on disk on a per-segment basis
 --		
 --------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free
 -- (
 --	dfsegment int,
 --	dfhostname text,

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
@@ -3358,6 +3358,70 @@ CreateStmt:	CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')'
 					n->policy = 0;
                     n->base.postCreate = NULL;
 
+					$$ = (Node *)n;
+				}
+		| CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')'
+		  ExtTypedesc FORMAT Sconst format_opt ext_opt_encoding_list OptDistributedBy OptTabPartitionBy
+		  		{
+		  			CreateExternalStmt *n = makeNode(CreateExternalStmt);
+		  			n->iswritable = TRUE;
+		  			n->isexternal = FALSE;
+		  			n->isweb = FALSE;
+		  			$4->istemp = $2;
+		  			n->base.relation = $4;
+		  			n->base.tableElts = $6;
+		  			n->exttypedesc = $8;
+		  			n->format = $10;
+		  			n->base.options = $11;
+		  			n->encoding = $12;
+		  			n->sreh = NULL;
+		  			n->base.distributedBy = $13;
+		  			n->base.partitionBy = $14;
+		  			n->policy = 0;
+
+		  			/* various syntax checks for internal table implemented using pluggable storage */
+		  			if(((ExtTableTypeDesc *) n->exttypedesc)->exttabletype != EXTTBL_TYPE_UNKNOWN)
+		  			{
+		  				ereport(ERROR,
+		  				        (errcode(ERRCODE_SYNTAX_ERROR),
+		  				         errmsg("LOCATION or EXECUTE may not be used for native table"),
+		  				         errhint("Use CREATE TABLE FORMAT or CREATE EXTERNAL TABLE instead"),
+		  				         errOmitLocation(true)));
+		  			}
+
+		  			ListCell *cell;
+		  			foreach (cell, n->base.tableElts)
+		  			{
+		  				Node *node = (Node *)lfirst(cell);
+		  				switch (nodeTag(node))
+		  				{
+		  					case T_ColumnDef:
+		  					{
+		  						if (((ColumnDef *)node)->constraints != NIL)
+		  						{
+									ereport(ERROR,
+									  (errcode(ERRCODE_SYNTAX_ERROR),
+									  errmsg("Do not support column constraint")));
+		  						}
+		  					}
+		  					break;
+
+		  					case T_Constraint:
+		  					{
+		  						if (((Constraint *)node)->contype != CONSTR_PRIMARY)
+		  						{
+		  							ereport(ERROR,
+		  							        (errcode(ERRCODE_SYNTAX_ERROR),
+		  							         errmsg("Do not support table constraint other than primary key")));
+		  						}
+		  					}
+		  					break;
+
+		  					default:
+		  						break;
+		  				}
+		  			}
+
 					$$ = (Node *)n;
 				}
 		;
@@ -4643,7 +4707,7 @@ CreateExternalStmt:	CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n
 
 OptWritable:	WRITABLE				{ $$ = TRUE; }
 				| READABLE				{ $$ = FALSE; }
-				| /*EMPTY*/				{ $$ = FALSE; }
+				| /*EMPTY*/				{ $$ = TRUE; }
 				;
 
 OptWeb:		WEB						{ $$ = TRUE; }
@@ -4671,6 +4735,16 @@ ExtTypedesc:
 
 				$$ = (Node *)n;
 			}
+			| /*EMPTY*/
+			{
+				ExtTableTypeDesc *n = makeNode(ExtTableTypeDesc);
+				n->exttabletype = EXTTBL_TYPE_UNKNOWN;
+				n->location_list = NIL;
+				n->command_string = NULL;
+				n->on_clause = NIL;
+
+				$$ = (Node *)n;
+			}
 			;
 
 ext_on_clause_list: