diff --git a/contrib/exthdfs/Makefile b/contrib/exthdfs/Makefile index e247664b96..7bf514f4e2 100644 --- a/contrib/exthdfs/Makefile +++ b/contrib/exthdfs/Makefile @@ -21,7 +21,7 @@ OBJS = exthdfs.o PG_CPPFLAGS = -I$(libpq_srcdir) PG_LIBS = $(libpq_pgport) -override CFLAGS += -lhdfs3 +override CFLAGS += -lhdfs3 -lstorage ifdef USE_PGXS PGXS := $(shell pg_config --pgxs) diff --git a/contrib/exthdfs/exthdfs.c b/contrib/exthdfs/exthdfs.c index 70e65109be..2c501ece5a 100644 --- a/contrib/exthdfs/exthdfs.c +++ b/contrib/exthdfs/exthdfs.c @@ -22,13 +22,11 @@ #include "common.h" #include "access/extprotocol.h" #include "cdb/cdbdatalocality.h" -#include "storage/fd.h" -#include "storage/filesystem.h" +#include "cdb/cdbfilesystemcredential.h" +#include "cdb/cdbvars.h" +#include "storage/cwrapper/hdfs-file-system-c.h" #include "utils/uri.h" - - - PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(hdfsprotocol_blocklocation); @@ -421,30 +419,47 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS) } /* SHOULD ADD LOGIC HERE TO CREATE UNEXISTING PATH */ - if (pvalidator_data->forceCreateDir) { - + if (pvalidator_data->forceCreateDir) + { elog(LOG, "hdfs_validator() forced creating dir"); - /* Create file system instance */ - hdfsFS fs = hdfsConnect(uri->hostname, uri->port); + /* Create file system instance */ + FscHdfsFileSystemC *fs = FscHdfsNewFileSystem(uri->hostname, + uri->port); if (fs == NULL) { elog(ERROR, "hdfsprotocol_validate : " - "failed to create HDFS instance to connect to %s:%d", - uri->hostname, uri->port); + "failed to create HDFS instance to connect to %s:%d", + uri->hostname, uri->port); } - if (hdfsExists(fs, uri->path) == -1) + if (FscHdfsExistPath(fs, uri->path) + && FscHdfsGetFileKind(fs, uri->path) == 'F') elog(ERROR, "hdfsprotocol_validate : " - "Location \"%s\" is not exist", - uri->path); + "Location \"%s\" is a file, not supported yet. " + "Only support directory now", + uri->path); + if (pvalidator_data->direction == EXT_VALIDATE_WRITE + && !FscHdfsExistInsertPath(fs, uri->path)) + { + elog(LOG, "hdfs_validator() to create url %s", uri->path); + FscHdfsCreateInsertDir(fs, uri->path); + if (FscHdfsHasErrorRaised(fs)) + { + CatchedError *ce = FscHdfsGetFileSystemError(fs); + elog(ERROR, "hdfsprotocol_validate : " + "failed to create directory %s : %s(%d)", + uri->path, + ce->errMessage, ce->errCode); + } + } - /* destroy fs instance */ - hdfsDisconnect(fs); + /* destroy fs instance */ + FscHdfsFreeFileSystemC(&fs); } /* Clean up temporarily created instances */ - FreeExternalTableUri(uri); + pfree(uri); if (nnaddr != NULL) { pfree(nnaddr); diff --git a/src/backend/catalog/gp_toolkit.sql.in b/src/backend/catalog/gp_toolkit.sql.in index a620284b7c..da3b75f125 100644 --- a/src/backend/catalog/gp_toolkit.sql.in +++ b/src/backend/catalog/gp_toolkit.sql.in @@ -179,7 +179,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public; -- External table that determines the local segment id -- -------------------------------------------------------------------------------- --- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid +-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid -- ( -- localid int -- ) @@ -196,7 +196,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public; -- External table that determines the master's segment id -- -------------------------------------------------------------------------------- --- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid +-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid -- ( -- masterid int -- ) @@ -238,7 +238,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public; -- External table to read segment log; requires superuser privilege -- -------------------------------------------------------------------------------- --- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext +-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext -- ( -- logtime timestamp with time zone, -- loguser text, @@ -285,7 +285,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public; -- External table to read the master log; requires superuser privilege -- -------------------------------------------------------------------------------- -CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext +CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext ( logtime timestamp with time zone, loguser text, @@ -319,7 +319,7 @@ CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext logstack text ) EXECUTE E'cat $GP_SEG_DATADIR/pg_log/*.csv' ON MASTER -FORMAT 'CSV' (DELIMITER AS ',' NULL AS '' QUOTE AS '"'); +FORMAT 'CSV' (DELIMITER ',' NULL '' QUOTE '"'); REVOKE ALL ON TABLE %%JETPACK_PREFIX_AUX%%log_master_ext FROM public; @@ -994,7 +994,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX%%stats_missing TO public; -- External table to determine free space on disk on a per-segment basis -- -------------------------------------------------------------------------------- --- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free +-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free -- ( -- dfsegment int, -- dfhostname text, diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b443bca5b0..4dbfcb2ac7 100755 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -3358,6 +3358,70 @@ CreateStmt: CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')' n->policy = 0; n->base.postCreate = NULL; + $$ = (Node *)n; + } + | CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')' + ExtTypedesc FORMAT Sconst format_opt ext_opt_encoding_list OptDistributedBy OptTabPartitionBy + { + CreateExternalStmt *n = makeNode(CreateExternalStmt); + n->iswritable = TRUE; + n->isexternal = FALSE; + n->isweb = FALSE; + $4->istemp = $2; + n->base.relation = $4; + n->base.tableElts = $6; + n->exttypedesc = $8; + n->format = $10; + n->base.options = $11; + n->encoding = $12; + n->sreh = NULL; + n->base.distributedBy = $13; + n->base.partitionBy = $14; + n->policy = 0; + + /* various syntax checks for internal table implemented using pluggable storage */ + if(((ExtTableTypeDesc *) n->exttypedesc)->exttabletype != EXTTBL_TYPE_UNKNOWN) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("LOCATION or EXECUTE may not be used for native table"), + errhint("Use CREATE TABLE FORMAT or CREATE EXTERNAL TABLE instead"), + errOmitLocation(true))); + } + + ListCell *cell; + foreach (cell, n->base.tableElts) + { + Node *node = (Node *)lfirst(cell); + switch (nodeTag(node)) + { + case T_ColumnDef: + { + if (((ColumnDef *)node)->constraints != NIL) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("Do not support column constraint"))); + } + } + break; + + case T_Constraint: + { + if (((Constraint *)node)->contype != CONSTR_PRIMARY) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("Do not support table constraint other than primary key"))); + } + } + break; + + default: + break; + } + } + $$ = (Node *)n; } ; @@ -4643,7 +4707,7 @@ CreateExternalStmt: CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n OptWritable: WRITABLE { $$ = TRUE; } | READABLE { $$ = FALSE; } - | /*EMPTY*/ { $$ = FALSE; } + | /*EMPTY*/ { $$ = TRUE; } ; OptWeb: WEB { $$ = TRUE; } @@ -4671,6 +4735,16 @@ ExtTypedesc: $$ = (Node *)n; } + | /*EMPTY*/ + { + ExtTableTypeDesc *n = makeNode(ExtTableTypeDesc); + n->exttabletype = EXTTBL_TYPE_UNKNOWN; + n->location_list = NIL; + n->command_string = NULL; + n->on_clause = NIL; + + $$ = (Node *)n; + } ; ext_on_clause_list: