From 3e020f991609bc4beab714d41c28c556a37ffa81 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 4 Sep 2024 10:06:46 -0700 Subject: [PATCH 01/14] CADC-13452 make action permission checking method static for use by non-action classes --- .../nrc/cadc/vosi/actions/DeleteAction.java | 2 +- .../ca/nrc/cadc/vosi/actions/GetAction.java | 4 +- .../vosi/actions/GetPermissionsAction.java | 4 +- .../vosi/actions/PostPermissionsAction.java | 4 +- .../ca/nrc/cadc/vosi/actions/PutAction.java | 2 +- .../nrc/cadc/vosi/actions/TablesAction.java | 230 ++++++++++++++---- 6 files changed, 186 insertions(+), 60 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/DeleteAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/DeleteAction.java index 226a8cec..1c326cc9 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/DeleteAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/DeleteAction.java @@ -109,7 +109,7 @@ public void doAction() throws Exception { ts.setDataSource(ds); tm = new DatabaseTransactionManager(ds); - checkDropTablePermission(ts, tableName); + TablesAction.checkDropTablePermission(ts, tableName, logInfo); try { diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java index b153ed08..3a9ef839 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetAction.java @@ -123,7 +123,7 @@ public void doAction() throws Exception { TapSchemaDAO dao = getTapSchemaDAO(); if (tableName != null) { - checkTableReadPermissions(dao, tableName); + checkTableReadPermissions(dao, tableName, logInfo); TableDesc td = dao.getTable(tableName); if (td == null) { // currently, permission check already threw this @@ -134,7 +134,7 @@ public void doAction() throws Exception { syncOutput.setHeader("Content-Type", "text/xml"); tw.write(td, new OutputStreamWriter(syncOutput.getOutputStream())); } else if (schemaName != null) { - checkViewSchemaPermissions(dao, schemaName); + checkViewSchemaPermissions(dao, schemaName, logInfo); // TODO: TapSchemaDAO only supports schema only, ok for detail=min // should at least list tables for default detail // should provide columns at detail=max diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java index 0dc8c49a..6a461693 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/GetPermissionsAction.java @@ -104,9 +104,9 @@ public void doAction() throws Exception { TapSchemaDAO dao = getTapSchemaDAO(); TapPermissions permissions = null; if (Util.isSchemaName(name)) { - permissions = checkViewSchemaPermissions(dao, name); + permissions = checkViewSchemaPermissions(dao, name, logInfo); } else if (Util.isTableName(name)) { - permissions = checkViewTablePermissions(dao, name); + permissions = checkViewTablePermissions(dao, name, logInfo); } else { throw new IllegalArgumentException("No such object: " + name); } diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PostPermissionsAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PostPermissionsAction.java index 59cfdb4e..fa59bf75 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PostPermissionsAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PostPermissionsAction.java @@ -116,13 +116,13 @@ public void doAction() throws Exception { // get the permissions first to ensure table exists and to ensure user // is allowed to view/modifiy permissions. if (Util.isSchemaName(name)) { - checkModifySchemaPermissions(dao, name); + TablesAction.checkModifySchemaPermissions(dao, name, logInfo); TapPermissions perms = dao.getSchemaPermissions(name); modifyPermissions(perms); log.debug("Setting schema permissions to: \n" + perms); dao.setSchemaPermissions(name, perms); } else if (Util.isTableName(name)) { - checkModifyTablePermissionsPermissions(dao, name); + TablesAction.checkModifyTablePermissions(dao, name, logInfo); TapPermissions perms = dao.getTablePermissions(name); modifyPermissions(perms); log.debug("Setting table permissions to: \n" + perms); diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java index 0146d85e..abf3d7c9 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java @@ -114,7 +114,7 @@ public void doAction() throws Exception { checkIsAdmin(); } else { // create table - checkSchemaWritePermissions(ts, schemaName); + TablesAction.checkSchemaWritePermissions(ts, schemaName, logInfo); } if (tableName != null) { diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java index 5601c9ee..c12710cf 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java @@ -168,7 +168,7 @@ String[] getTarget() throws ResourceNotFoundException { /** * Create and configure a TapSchemaDAO instance. * - * @return + * @return a TapSchemaDAO instance. */ protected final TapSchemaDAO getTapSchemaDAO() { PluginFactory pf = new PluginFactory(); @@ -178,11 +178,21 @@ protected final TapSchemaDAO getTapSchemaDAO() { dao.setOrdered(true); return dao; } - - // schema owner can drop - // table owner can drop - // no group permissions used - void checkDropTablePermission(TapSchemaDAO dao, String tableName) + + /** + * Check if the calling user has permission to drop the specified table from the TAP schema. + * A user can drop a table if: + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for drop permission + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have drop permissions for the table. + * @throws ResourceNotFoundException if the table or table's schema are not found in the TAP schema. + */ + public static void checkDropTablePermission(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) throws AccessControlException, ResourceNotFoundException { String schemaName = Util.getSchemaFromTable(tableName); @@ -195,18 +205,33 @@ void checkDropTablePermission(TapSchemaDAO dao, String tableName) throw new ResourceNotFoundException("table not found: " + tableName); } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("drop table allowed: schema owner"); + logInfo.setMessage("drop table allowed: schema owner"); return; } if (Util.isOwner(tablePermissions)) { - super.logInfo.setMessage("drop table allowed: table owner"); + logInfo.setMessage("drop table allowed: table owner"); return; } throw new AccessControlException("permission denied"); } - - // schema owner can view schema permissions - TapPermissions checkViewSchemaPermissions(TapSchemaDAO dao, String schemaName) + + /** + * Check if the calling user has view permissions for the specified schema. + * A user can view the permissions for a schema if ons the following is true: + * + * @param dao DAO for the TAP schema + * @param schemaName the schema to check for view permission + * @param logInfo webservice logging + * @return the TapPermissions for the specified schema. + * @throws AccessControlException if the user does not have view permissions for the schema. + * @throws ResourceNotFoundException if the schema is not found in the TAP schema. + */ + TapPermissions checkViewSchemaPermissions(TapSchemaDAO dao, String schemaName, WebServiceLogInfo logInfo) throws AccessControlException, ResourceNotFoundException { TapPermissions schemaPermissions = dao.getSchemaPermissions(schemaName); @@ -214,26 +239,40 @@ TapPermissions checkViewSchemaPermissions(TapSchemaDAO dao, String schemaName) throw new ResourceNotFoundException("schema not found: " + schemaName); } if (schemaPermissions.owner == null) { - super.logInfo.setMessage("view table allowed: null schema owner"); + logInfo.setMessage("view table allowed: null schema owner"); return schemaPermissions; } if (schemaPermissions.isPublic) { - super.logInfo.setMessage("view table allowed: public schema"); + logInfo.setMessage("view table allowed: public schema"); return schemaPermissions; } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("view schema permissions allowed: schema owner"); + logInfo.setMessage("view schema permissions allowed: schema owner"); return schemaPermissions; } if (checkIsAdmin()) { - super.logInfo.setMessage("view schema permissions allowed: admin"); + logInfo.setMessage("view schema permissions allowed: admin"); return schemaPermissions; } throw new AccessControlException("permission denied"); } - - // schema owner can modify schema permissions - void checkModifySchemaPermissions(TapSchemaDAO dao, String schemaName) + + /** + * Check if the calling user has permission to modify permissions for the specified schema. + * A user has permission to modify a schema's permissions if: + * + * A user cannot update schema permissions if they are a member of a schema's read-write group + * because they could remove themselves from the read-write group and lose access the the schema. + * + * @param dao DAO for the TAP schema + * @param schemaName the schema to check for modify permissions + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have permission to modify the schema's permissions. + * @throws ResourceNotFoundException if the schema is not found in the TAP schema. + */ + public static void checkModifySchemaPermissions(TapSchemaDAO dao, String schemaName, WebServiceLogInfo logInfo) throws AccessControlException, ResourceNotFoundException { TapPermissions schemaPermissions = dao.getSchemaPermissions(schemaName); @@ -241,14 +280,28 @@ void checkModifySchemaPermissions(TapSchemaDAO dao, String schemaName) throw new ResourceNotFoundException("schema not found: " + schemaName); } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("modify schema permissions allowed: schema owner"); + logInfo.setMessage("modify schema permissions allowed: schema owner"); return; } throw new AccessControlException("permission denied"); } - - // schema owner and table owner can view table permissions - TapPermissions checkViewTablePermissions(TapSchemaDAO dao, String tableName) + + /** + * Check if the calling user has permissions to view the permissions for the specified table. + * A user has permission to view a table's permissions if ons the following is true: + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for view permission + * @param logInfo webservice logging + * @return the TapPermissions for the specified table. + * @throws AccessControlException if the user does not have permission to view the table's permissions. + * @throws ResourceNotFoundException if the table is not found in the TAP schema. + */ + public TapPermissions checkViewTablePermissions(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) throws AccessControlException, ResourceNotFoundException { String schemaName = Util.getSchemaFromTable(tableName); @@ -262,22 +315,37 @@ TapPermissions checkViewTablePermissions(TapSchemaDAO dao, String tableName) throw new ResourceNotFoundException("table not found: " + tableName); } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("view table permissions allowed: schema owner"); + logInfo.setMessage("view table permissions allowed: schema owner"); return tablePermissions; } if (Util.isOwner(tablePermissions)) { - super.logInfo.setMessage("view table permissions allowed: table owner"); + logInfo.setMessage("view table permissions allowed: table owner"); return tablePermissions; } if (checkIsAdmin()) { - super.logInfo.setMessage("view table permissions allowed: admin"); + logInfo.setMessage("view table permissions allowed: admin"); return tablePermissions; } throw new AccessControlException("permission denied"); } - - // schema owner and table owner can modify table permissions - void checkModifyTablePermissionsPermissions(TapSchemaDAO dao, String tableName) + + /** + * Check if the calling user has permission to modify permissions for the specified table. + * A user has permission to modify a table's permissions if: + * + * A user cannot update table permissions if they are a member of a table's read-write group + * because they could remove themselves from the read-write group and lose access the the table. + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for modify permission + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have permission to modify the table's permissions. + * @throws ResourceNotFoundException if the table or table's schema is not found in the TAP schema. + */ + public static void checkModifyTablePermissions(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) throws AccessControlException, ResourceNotFoundException { String schemaName = Util.getSchemaFromTable(tableName); @@ -290,19 +358,39 @@ void checkModifyTablePermissionsPermissions(TapSchemaDAO dao, String tableName) throw new ResourceNotFoundException("table not found: " + tableName); } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("modify table permissions allowed: schema owner"); + logInfo.setMessage("modify table permissions allowed: schema owner"); return; } if (Util.isOwner(tablePermissions)) { - super.logInfo.setMessage("modify table permissions allowed: table owner"); + logInfo.setMessage("modify table permissions allowed: table owner"); return; } throw new AccessControlException("permission denied"); } - - // if anon or authenticated check public - // if authenticated check schema and table owners, readGroups, readWriteGroups - void checkTableReadPermissions(TapSchemaDAO dao, String tableName) + + /** + * Check is the calling user has read permission for a table. + * A user has read permission for a table if one of the following is true: + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for read permission + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have read permission to the table. + * @throws IOException if there is an error retrieving group memberships. + * @throws InterruptedException if there is an error querying for group membership. + * @throws ResourceNotFoundException if the table is not found in the TAP schema. + */ + public void checkTableReadPermissions(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) throws AccessControlException, IOException, InterruptedException, ResourceNotFoundException { TapPermissions tablePermissions = dao.getTablePermissions(tableName); @@ -317,33 +405,33 @@ void checkTableReadPermissions(TapSchemaDAO dao, String tableName) throw new ResourceNotFoundException("schema not found: " + schemaName); } if (schemaPermissions.owner == null) { - super.logInfo.setMessage("view table allowed: null schema owner"); + logInfo.setMessage("view table allowed: null schema owner"); return; } if (schemaPermissions.isPublic) { - super.logInfo.setMessage("view table allowed: public schema"); + logInfo.setMessage("view table allowed: public schema"); return; } if (tablePermissions.owner == null) { - super.logInfo.setMessage("view table allowed: null table owner"); + logInfo.setMessage("view table allowed: null table owner"); return; } if (tablePermissions.isPublic) { - super.logInfo.setMessage("view table allowed: public table"); + logInfo.setMessage("view table allowed: public table"); return; } if (Util.isOwner(tablePermissions)) { - super.logInfo.setMessage("view table allowed: table owner"); + logInfo.setMessage("view table allowed: table owner"); return; } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("view table allowed: schema owner"); + logInfo.setMessage("view table allowed: schema owner"); return; } if (checkIsAdmin()) { - super.logInfo.setMessage("view table allowed: admin"); + logInfo.setMessage("view table allowed: admin"); return; } @@ -367,22 +455,47 @@ void checkTableReadPermissions(TapSchemaDAO dao, String tableName) GroupURI permittingGroup = Util.getPermittedGroup(groupClient, readGroups); if (permittingGroup != null) { - super.logInfo.setMessage("view table allowed: member of group " + permittingGroup); + logInfo.setMessage("view table allowed: member of group " + permittingGroup); return; } throw new AccessControlException("permission denied"); } - + + /** + * Check if the calling user has write permission to the specified table. + * A user has write permission to a table if one of the following is true: + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for write permissions + * @throws AccessControlException if the user does not have write permission to the table. + * @throws IOException if there is an error retrieving group memberships. + * @throws ResourceNotFoundException if the table is not found in the TAP schema. + */ public void checkTableWritePermissions(TapSchemaDAO dao, String tableName) throws AccessControlException, IOException, ResourceNotFoundException { TablesAction.checkTableWritePermissions(dao, tableName, logInfo); } - - // if authenticated table owners, readWriteGroup members - // static method here so that TableUpdateRunner can make this call - static void checkTableWritePermissions(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) + + /** + * Check if the calling user has write permission to the specified table. + * A user has write permission to a table if one of the following is true: + * + * @param dao DAO for the TAP schema + * @param tableName the table to check for write permissions + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have write permission to the table. + * @throws IOException if there is an error retrieving group memberships. + * @throws ResourceNotFoundException if the table is not found in the TAP schema. + */ + public static void checkTableWritePermissions(TapSchemaDAO dao, String tableName, WebServiceLogInfo logInfo) throws AccessControlException, IOException, ResourceNotFoundException { TapPermissions tablePermissions = dao.getTablePermissions(tableName); @@ -405,9 +518,22 @@ static void checkTableWritePermissions(TapSchemaDAO dao, String tableName, WebSe } throw new AccessControlException("permission denied"); } - - // if authenticated check schema owner and readWriteGroup - void checkSchemaWritePermissions(TapSchemaDAO dao, String schemaName) + + /** + * Check if the calling user has write permission to the specified schema. + * A user has write permission to a schema if one of the following is true: + * + * @param dao DAO for the TAP schema + * @param schemaName the schema to check for write permissions + * @param logInfo webservice logging + * @throws AccessControlException if the user does not have write permission to the schema. + * @throws IOException if there is an error retrieving group memberships. + * @throws ResourceNotFoundException if the schema is not found in the TAP schema. + */ + public static void checkSchemaWritePermissions(TapSchemaDAO dao, String schemaName, WebServiceLogInfo logInfo) throws AccessControlException, IOException, ResourceNotFoundException { TapPermissions schemaPermissions = dao.getSchemaPermissions(schemaName); @@ -415,7 +541,7 @@ void checkSchemaWritePermissions(TapSchemaDAO dao, String schemaName) throw new ResourceNotFoundException("not found: " + schemaName); } if (Util.isOwner(schemaPermissions)) { - super.logInfo.setMessage("schema write allowed: schema owner"); + logInfo.setMessage("schema write allowed: schema owner"); return; } final IvoaGroupClient groupClient = new IvoaGroupClient(); @@ -424,7 +550,7 @@ void checkSchemaWritePermissions(TapSchemaDAO dao, String schemaName) permittedGroups.add(schemaPermissions.readWriteGroup); GroupURI permittedGroup = Util.getPermittedGroup(groupClient, permittedGroups); if (permittedGroup != null) { - super.logInfo.setMessage("schema write allowed: member of table group " + permittedGroup); + logInfo.setMessage("schema write allowed: member of table group " + permittedGroup); return; } } From 59cb7681b283abd965a4a5aca41bef64085808c5 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 4 Sep 2024 10:13:47 -0700 Subject: [PATCH 02/14] CADC-13452 add mapping from database data type to TAPDataType --- .../nrc/cadc/tap/db/BasicDataTypeMapper.java | 41 ++++++++++++++++++- .../ca/nrc/cadc/tap/db/DatabaseDataType.java | 9 ++++ cadc-tap-server-pg/build.gradle | 4 +- .../cadc/tap/pg/PostgresDataTypeMapper.java | 22 ++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/BasicDataTypeMapper.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/BasicDataTypeMapper.java index 558c2528..a00caa75 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/BasicDataTypeMapper.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/BasicDataTypeMapper.java @@ -118,6 +118,11 @@ public String toString() { */ protected final Map dataTypes = new HashMap<>(); + /** + * Mapping of database data types to VOTable data types. + */ + protected final Map dbDataTypes = new HashMap<>(); + public BasicDataTypeMapper() { // votable type -> db type dataTypes.put(TapDataType.BOOLEAN, new TypePair("BOOLEAN", Types.BOOLEAN)); @@ -131,6 +136,25 @@ public BasicDataTypeMapper() { dataTypes.put(TapDataType.STRING, new TypePair("CHAR", Types.CHAR)); dataTypes.put(TapDataType.TIMESTAMP, new TypePair("TIMESTAMP", Types.TIMESTAMP)); dataTypes.put(TapDataType.URI, new TypePair("CHAR", Types.CHAR)); + + // DatabaseMetadata -> TAP_DATA_TYPE + // TYPE_NAME DATA_TYPE TAP_DATA_TYPE + // bool -7 BOOLEAN + // bpchar 1 CHAR + // varchar 4096 12 STRING or URI + // int2 5 SHORT + // int4 4 INTEGER + // int8 -5 LONG + // float4 7 FLOAT + // float8 8 DOUBLE + // timestamp 93 TIMESTAMP + dbDataTypes.put("varchar", TapDataType.STRING); + dbDataTypes.put("int2", TapDataType.SHORT); + dbDataTypes.put("int4", TapDataType.INTEGER); + dbDataTypes.put("int8", TapDataType.LONG); + dbDataTypes.put("float4", TapDataType.FLOAT); + dbDataTypes.put("float8", TapDataType.DOUBLE); + dbDataTypes.put("timestamp", TapDataType.TIMESTAMP); } /** @@ -189,8 +213,21 @@ public String getIndexUsingQualifier(ColumnDesc columnDesc, boolean unique) { public String getIndexColumnOperator(ColumnDesc columnDesc) { return null; } - - + + /** + * Maps standard database datatypes to a TapDatatype. Database specific datatypes can by mapped in a sub class database specific mapper. + * + * @param datatype database datatype + * @return TapDatatype + */ + public TapDataType getTapDataType(String datatype) { + TapDataType tapDataType = dbDataTypes.get(datatype); + if (tapDataType != null) { + return tapDataType; + } + throw new UnsupportedOperationException("Unknown database datatype: " + datatype); + } + /** * Find or create a TypePair for the specified data type. The current implementation * looks for exact matches in the dataTypes map and, if not found, it rechecks with diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/DatabaseDataType.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/DatabaseDataType.java index 8e1c46ed..81d21dd5 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/DatabaseDataType.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/DatabaseDataType.java @@ -74,6 +74,7 @@ import ca.nrc.cadc.dali.Point; import ca.nrc.cadc.dali.Polygon; import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.TapDataType; /** * Interface to convert ADQL data types to a database @@ -182,4 +183,12 @@ public interface DatabaseDataType Object getArrayObject(float[] val); Object getArrayObject(double[] val); + + /** + * Convert a database data type to a a TAP data type. + * @param datatype the database data type + * @return a TapDataType + */ + TapDataType getTapDataType(String datatype); + } diff --git a/cadc-tap-server-pg/build.gradle b/cadc-tap-server-pg/build.gradle index ca1f015c..48ede01f 100644 --- a/cadc-tap-server-pg/build.gradle +++ b/cadc-tap-server-pg/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '1.1.0' +version = '1.1.1' description = 'OpenCADC TAP-1.1 tap server plugin (PostgreSQL+pgsphere)' def git_url = 'https://github.com/opencadc/tap' @@ -24,7 +24,7 @@ def git_url = 'https://github.com/opencadc/tap' dependencies { compile 'org.opencadc:cadc-dali-pg:[0.1,)' compile 'org.opencadc:cadc-adql:[1.1.14,)' - compile 'org.opencadc:cadc-tap-schema:[1.1.22, )' + compile 'org.opencadc:cadc-tap-schema:[1.1.34, )' compile 'org.opencadc:cadc-tap-server:[1.1.7, )' testCompile 'junit:junit:[4.0,5.0)' diff --git a/cadc-tap-server-pg/src/main/java/ca/nrc/cadc/tap/pg/PostgresDataTypeMapper.java b/cadc-tap-server-pg/src/main/java/ca/nrc/cadc/tap/pg/PostgresDataTypeMapper.java index 08250e52..10d8f823 100644 --- a/cadc-tap-server-pg/src/main/java/ca/nrc/cadc/tap/pg/PostgresDataTypeMapper.java +++ b/cadc-tap-server-pg/src/main/java/ca/nrc/cadc/tap/pg/PostgresDataTypeMapper.java @@ -110,6 +110,27 @@ public PostgresDataTypeMapper() { dataTypes.put(new TapDataType("long", "*", null), new TypePair("bigint[]", null)); dataTypes.put(new TapDataType("float", "*", null), new TypePair("real[]", null)); dataTypes.put(new TapDataType("double", "*", null), new TypePair("double precision[]", null)); + + // DatabaseMetadata -> TAP_DATA_TYPE + // TYPE_NAME DATA_TYPE TAP_DATA_TYPE + // polygon 1111 INTERVAL + // spoint 1111 POINT + // scircle 1111 CIRCLE + // spoly 1111 POLYGON + // _int2 2003 short[] + // _int4 2003 int[] + // _int8 2003 long[] + // _float4 2003 float[] + // _float8 2003 double[] + dbDataTypes.put("polygon", TapDataType.INTERVAL); + dbDataTypes.put("spoint", TapDataType.POINT); + dbDataTypes.put("scircle", TapDataType.CIRCLE); + dbDataTypes.put("spoly", TapDataType.POLYGON); + dbDataTypes.put("_int2", new TapDataType("short", "*", null)); + dbDataTypes.put("_int4", new TapDataType("int", "*", null)); + dbDataTypes.put("_int8", new TapDataType("long", "*", null)); + dbDataTypes.put("_float4", new TapDataType("float", "*", null)); + dbDataTypes.put("_float8", new TapDataType("double", "*", null)); } @Override @@ -315,4 +336,5 @@ public Object getArrayObject(long[] val) { throw new RuntimeException("BUG: failed to convert long[] to PGobject", ex); } } + } From 8e7368e32b584888becb4fe474f7163ba470618c Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 4 Sep 2024 10:14:43 -0700 Subject: [PATCH 03/14] CADC-13452 new TableIngester to copy database table metadata to tap_schema --- cadc-tap-schema/build.gradle | 2 +- .../ca/nrc/cadc/tap/db/TableIngesterTest.java | 198 ++++++++++++++ .../ca/nrc/cadc/tap/db/TableIngester.java | 219 ++++++++++++++++ .../cadc/vosi/actions/TableUpdateRunner.java | 247 +++++++++++------- .../ca/nrc/cadc/tap/pg/TableIngesterTest.java | 201 ++++++++++++++ 5 files changed, 770 insertions(+), 97 deletions(-) create mode 100644 cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java create mode 100644 cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java create mode 100644 cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java diff --git a/cadc-tap-schema/build.gradle b/cadc-tap-schema/build.gradle index d290c8b8..aeffa024 100644 --- a/cadc-tap-schema/build.gradle +++ b/cadc-tap-schema/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '1.1.33' +version = '1.1.34' description = 'OpenCADC TAP-1.1 tap schema server library' def git_url = 'https://github.com/opencadc/tap' diff --git a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java new file mode 100644 index 00000000..6ec6c899 --- /dev/null +++ b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java @@ -0,0 +1,198 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package ca.nrc.cadc.tap.db; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import ca.nrc.cadc.util.Log4jInit; +import java.util.List; +import java.util.concurrent.Callable; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +public class TableIngesterTest { + private static final Logger log = Logger.getLogger(TableIngesterTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.tap.db", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.tap.schema", Level.DEBUG); + } + + private final DataSource dataSource; + private final TapSchemaDAO tapSchemaDAO; + private final String TEST_SCHEMA = "int_test_schema"; + + public TableIngesterTest() { + // create a datasource and register with JNDI + try { + DBConfig conf = new DBConfig(); + ConnectionConfig cc = conf.getConnectionConfig("TAP_SCHEMA_TEST", "cadctest"); + this.dataSource = DBUtil.getDataSource(cc); + log.info("configured data source: " + cc.getServer() + "," + cc.getDatabase() + "," + cc.getDriver() + "," + cc.getURL()); + this.tapSchemaDAO = new TapSchemaDAO(); + this.tapSchemaDAO.setDataSource(this.dataSource); + } catch (Exception ex) { + log.error("setup failed", ex); + throw new IllegalStateException("failed to create DataSource", ex); + } + } + + @Test + public void testTableIngest() { + String testTable = TEST_SCHEMA + ".testTableIngest"; + TableCreator tableCreator = new TableCreator(dataSource); + try { + // cleanup + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + // ingest table into the tap_schema + TableIngester tableIngester = new TableIngester(dataSource); + tableIngester.ingest(TEST_SCHEMA, testTable); + log.debug("ingested table"); + + // check + + // compare database and tap_schema + SchemaDesc schemaDesc = tapSchemaDAO.getSchema(TEST_SCHEMA, true); + Assert.assertNotNull("schema", schemaDesc); + + TableDesc tableDesc = tapSchemaDAO.getTable(testTable); + Assert.assertNotNull("table", tableDesc); + + List databaseColumns = ingestTable.getColumnDescs(); + List tapSchemaColumns = tableDesc.getColumnDescs(); + for (ColumnDesc databaseColumn: databaseColumns) { + boolean found = false; + log.debug("database column: " + databaseColumn.getColumnName()); + for (ColumnDesc tapSchemaColumn : tapSchemaColumns) { + log.debug("tap_schema column: " + tapSchemaColumn.getColumnName()); + if (databaseColumn.getColumnName().equals(tapSchemaColumn.getColumnName())) { + Assert.assertEquals("datatype", databaseColumn.getDatatype(), tapSchemaColumn.getDatatype()); + found = true; + break; + } + } + Assert.assertTrue("tap_schema column not found: " + databaseColumn, found); + } + + // cleanup + try { +// tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + try { +// tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + TableDesc getTableDesc(String schemaName, String tableName) throws Exception { + final TableDesc tableDesc = new TableDesc(schemaName, tableName); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c0", TapDataType.STRING)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c1", TapDataType.SHORT)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c2", TapDataType.INTEGER)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c3", TapDataType.LONG)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c4", TapDataType.FLOAT)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c5", TapDataType.DOUBLE)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c6", TapDataType.TIMESTAMP)); + return tableDesc; + } + +} diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java new file mode 100644 index 00000000..f67f8f88 --- /dev/null +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java @@ -0,0 +1,219 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package ca.nrc.cadc.tap.db; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.db.DatabaseTransactionManager; +import ca.nrc.cadc.tap.PluginFactory; +import ca.nrc.cadc.tap.schema.ADQLIdentifierException; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import ca.nrc.cadc.tap.schema.TapSchemaUtil; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import javax.security.auth.Subject; +import javax.sql.DataSource; +import org.apache.log4j.Logger; + +public class TableIngester { + private static final Logger log = Logger.getLogger(TableIngester.class); + + static final String TAP_SCHEMA = "tap_schema"; + + private final DataSource dataSource; + private final DatabaseDataType databaseDataType; + private final TapSchemaDAO tapSchemaDAO; + + public TableIngester(DataSource dataSource) { + this.dataSource = dataSource; + PluginFactory pluginFactory = new PluginFactory(); + this.tapSchemaDAO = pluginFactory.getTapSchemaDAO(); + this.tapSchemaDAO.setDataSource(dataSource); + this.databaseDataType = pluginFactory.getDatabaseDataType(); + log.debug("loaded: " + databaseDataType.getClass().getName()); + } + + public void ingest(String schemaName, String tableName) { + // create the table description + TableDesc ingestTable; + try { + ingestTable = createTableDesc(schemaName, tableName); + } catch (SQLException e) { + throw new IllegalArgumentException(String.format("error getting database metadata for %s because: %s", + tableName, e.getMessage())); + } + + // check the table is valid ADQL + try { + TapSchemaUtil.checkValidTableName(ingestTable.getTableName()); + } catch (ADQLIdentifierException ex) { + throw new IllegalArgumentException("invalid table name: " + ingestTable.getTableName(), ex); + } + try { + for (ColumnDesc cd : ingestTable.getColumnDescs()) { + TapSchemaUtil.checkValidIdentifier(cd.getColumnName()); + } + } catch (ADQLIdentifierException ex) { + throw new IllegalArgumentException(ex.getMessage()); + } + + // make caller the table owner + Subject caller = AuthenticationUtil.getCurrentSubject(); + TapPermissions tapPermissions = new TapPermissions(); + tapPermissions.owner = caller; + ingestTable.tapPermissions = tapPermissions; + + DatabaseTransactionManager tm = new DatabaseTransactionManager(dataSource); + try { + tm.startTransaction(); + + // add the schema to the tap_schema if it doesn't exist + SchemaDesc schemaDesc = tapSchemaDAO.getSchema(schemaName, true); + if (schemaDesc == null) { + schemaDesc = new SchemaDesc(schemaName); + schemaDesc.tapPermissions = tapPermissions; + tapSchemaDAO.put(schemaDesc); + log.debug(String.format("added schema '%s' to tap_schema", schemaDesc.getSchemaName())); + } else { + log.debug(String.format("existing schema '%s' in tap_schema", schemaDesc.getSchemaName())); + } + + // add the table to the tap_schema + TableDesc tableDesc = tapSchemaDAO.getTable(tableName, true); + if (tableDesc != null) { + throw new IllegalStateException(String.format("table already exists in tap_schema: %s", tableName)); + } + tapSchemaDAO.put(ingestTable); + log.debug(String.format("added table '%s' to tap_schema", tableName)); + + tm.commitTransaction(); + } catch (Exception ex) { + try { + log.error("update tap_schema failed - rollback", ex); + tm.rollbackTransaction(); + log.error("update tap_schema failed - rollback: OK"); + } catch (Exception oops) { + log.error("update tap_schema failed - rollback : FAIL", oops); + } + throw new RuntimeException(String.format("failed to update tap_schema with %s", tableName), ex); + } finally { + if (tm.isOpen()) { + log.error("BUG: open transaction in finally - trying to rollback"); + try { + tm.rollbackTransaction(); + log.error("BUG: rollback in finally: OK"); + } catch (Exception oops) { + log.error("BUG: rollback in finally: FAIL", oops); + } + throw new RuntimeException("BUG: open transaction in finally"); + } + } + } + + protected TableDesc createTableDesc(String schemaName, String tableName) + throws SQLException { + log.debug(String.format("creating TableDesc for %s %s", schemaName, tableName)); + // get the table metadata + DatabaseMetaData databaseMetaData = dataSource.getConnection().getMetaData(); + //TODO too pg specific? table names are stored lower case in the system tables queried for the metadata + ResultSet columnInfo = databaseMetaData.getColumns(null, schemaName, "testtableingest", null); + ResultSet indexInfo = databaseMetaData.getIndexInfo(null, schemaName, "testtableingest", false, false); + // get column names for indexed columns + List indexedColumns = new ArrayList(); + while (indexInfo.next()) { + String indexedColumn = indexInfo.getString("COLUMN_NAME"); + indexedColumns.add(indexedColumn); + log.debug("indexed column: " + indexedColumn); + } + + // build TableDesc + TableDesc tableDesc = new TableDesc(TAP_SCHEMA, tableName); + tableDesc.tableType = TableDesc.TableType.TABLE; + log.debug(String.format("creating TableDesc %s %s", TAP_SCHEMA, tableName)); + while (columnInfo.next()) { + String columnName = columnInfo.getString("COLUMN_NAME"); + String columnType = columnInfo.getString("TYPE_NAME"); + TapDataType tapDataType = this.databaseDataType.getTapDataType(columnType); + log.debug(String.format("creating ColumnDesc %s %s %s", tableName, columnName, tapDataType)); + ColumnDesc columnDesc = new ColumnDesc(tableName, columnName, tapDataType); + columnDesc.indexed = indexedColumns.contains(columnName); + tableDesc.getColumnDescs().add(columnDesc); + } + return tableDesc; + } + +} diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java index 683c3643..743b5fba 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java @@ -75,7 +75,9 @@ import ca.nrc.cadc.rest.SyncOutput; import ca.nrc.cadc.tap.PluginFactory; import ca.nrc.cadc.tap.db.TableCreator; +import ca.nrc.cadc.tap.db.TableIngester; import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; import ca.nrc.cadc.tap.schema.TableDesc; import ca.nrc.cadc.tap.schema.TapSchemaDAO; import ca.nrc.cadc.uws.ErrorSummary; @@ -85,6 +87,7 @@ import ca.nrc.cadc.uws.server.JobRunner; import ca.nrc.cadc.uws.server.JobUpdater; import ca.nrc.cadc.uws.util.JobLogInfo; +import java.io.IOException; import java.security.AccessControlException; import java.util.ArrayList; import java.util.Date; @@ -120,6 +123,7 @@ public class TableUpdateRunner implements JobRunner { PARAM_NAMES.add("table"); PARAM_NAMES.add("index"); PARAM_NAMES.add("unique"); + PARAM_NAMES.add("op"); } private JobUpdater jobUpdater; @@ -169,108 +173,23 @@ private void doit() { logInfo.setMessage("Could not set job phase to EXECUTING."); return; } - log.debug(job.getID() + ": QUEUED -> EXECUTING [OK]"); + // check for the requested operation ParamExtractor pe = new ParamExtractor(PARAM_NAMES); Map> params = pe.getParameters(job.getParameterList()); - String tableName = getSingleValue("table", params); - String columnName = getSingleValue("index", params); - boolean unique = "true".equals(getSingleValue("unique", params)); - - // TODO: make create index optional and check for table load from URI params - - if (tableName == null) { - throw new IllegalArgumentException("missing parameter 'table'"); - } - if (columnName == null) { - throw new IllegalArgumentException("missing parameter 'index'"); - } - - PluginFactory pf = new PluginFactory(); - TapSchemaDAO ts = pf.getTapSchemaDAO(); - DataSource ds = getDataSource(); - ts.setDataSource(ds); - try { - log.debug("Checking table write permission"); - TablesAction.checkTableWritePermissions(ts, tableName, logInfo); - } catch (ResourceNotFoundException ex) { - throw new IllegalArgumentException("table not found: " + tableName); - } - - TableDesc td = ts.getTable(tableName); - if (td == null) { - // if this was not thrown in permission check above then we have an inconsistency between - // the tap_schema content and the table ownership - log.error("INCONSISTENT STATE: permission check says table " + tableName + "exists but it is not in tap_schema"); - throw new IllegalArgumentException("table not found: " + tableName); - } - - ColumnDesc cd = td.getColumn(columnName); - if (cd == null) { - throw new IllegalArgumentException("column not found: " + columnName + " in table " + tableName); - } - - if (cd.indexed) { - throw new IllegalArgumentException("column is already indexed: " + columnName + " in table " + tableName); + String op = getSingleValue("op", params); + if (op == null) { + throw new IllegalArgumentException("missing parameter 'op'"); } - DatabaseTransactionManager tm = new DatabaseTransactionManager(ds); - try { - tm.startTransaction(); - - // create index - TableCreator tc = new TableCreator(ds); - tc.createIndex(cd, unique); - - // createIndex can take consierable time so our view of the column metadata could be out of date - - // write lock row in tap_schema.columns - ts.put(cd); - - // get current values in case another thread has updated it - cd = ts.getColumn(tableName, cd.getColumnName()); - - // update tap_schema - cd.indexed = true; - ts.put(cd); - - tm.commitTransaction(); - } catch (Exception ex) { - boolean dbg = false; - if (ex instanceof IllegalArgumentException || ex instanceof UnsupportedOperationException) { - dbg = true; - } - try { - if (dbg) { - log.debug("create index and update tap_schema failed - rollback", ex); - } else { - log.error("create index and update tap_schema failed - rollback", ex); - } - tm.rollbackTransaction(); - if (dbg) { - log.debug("create index and update tap_schema failed - rollback: OK"); - } else { - log.error("create index and update tap_schema failed - rollback: OK"); - } - } catch (Exception oops) { - log.error("create index and update tap_schema - rollback : FAIL", oops); - } - if (ex instanceof IllegalArgumentException) { - throw ex; - } - throw new RuntimeException("failed to update table " + tableName + " reason: " + ex.getMessage(), ex); - } finally { - if (tm.isOpen()) { - log.error("BUG: open transaction in finally - trying to rollback"); - try { - tm.rollbackTransaction(); - log.error("BUG: rollback in finally: OK"); - } catch (Exception oops) { - log.error("BUG: rollback in finally: FAIL", oops); - } - throw new RuntimeException("BUG: open transaction in finally"); - } + // index or ingest table + if (op.equals("index")) { + indexTable(params); + } else if (op.equals("ingest")) { + ingestTable(params); + } else { + throw new IllegalArgumentException("unknown 'op' parameter: " + op); } ep = jobUpdater.setPhase(job.getID(), ExecutionPhase.EXECUTING, ExecutionPhase.COMPLETED, new Date()); @@ -305,6 +224,142 @@ private void doit() { } } + /** + * + * @param params + */ + protected void indexTable(Map> params) { + String tableName = getSingleValue("table", params); + String columnName = getSingleValue("column", params); + boolean unique = "true".equals(getSingleValue("unique", params)); + + if (tableName == null) { + throw new IllegalArgumentException("missing parameter 'table'"); + } + if (columnName == null) { + throw new IllegalArgumentException("missing parameter 'column'"); + } + + PluginFactory pf = new PluginFactory(); + TapSchemaDAO ts = pf.getTapSchemaDAO(); + DataSource ds = getDataSource(); + ts.setDataSource(ds); + try { + log.debug("Checking table write permission"); + TablesAction.checkTableWritePermissions(ts, tableName, logInfo); + } catch (ResourceNotFoundException | IOException ex) { + throw new IllegalArgumentException("table not found: " + tableName); + } + + TableDesc td = ts.getTable(tableName); + if (td == null) { + // if this was not thrown in permission check above then we have an inconsistency between + // the tap_schema content and the table ownership + log.error("INCONSISTENT STATE: permission check says table " + tableName + "exists but it is not in tap_schema"); + throw new IllegalArgumentException("table not found: " + tableName); + } + + ColumnDesc cd = td.getColumn(columnName); + if (cd == null) { + throw new IllegalArgumentException("column not found: " + columnName + " in table " + tableName); + } + if (cd.indexed) { + throw new IllegalArgumentException("column is already indexed: " + columnName + " in table " + tableName); + } + + DatabaseTransactionManager tm = new DatabaseTransactionManager(ds); + try { + tm.startTransaction(); + + // create index + TableCreator tc = new TableCreator(ds); + tc.createIndex(cd, unique); + + // createIndex can take considerable time so our view of the column metadata could be out of date + + // write lock row in tap_schema.columns + ts.put(cd); + + // get current values in case another thread has updated it + cd = ts.getColumn(tableName, cd.getColumnName()); + + // update tap_schema + cd.indexed = true; + ts.put(cd); + + tm.commitTransaction(); + } catch (Exception ex) { + boolean dbg = false; + if (ex instanceof IllegalArgumentException || ex instanceof UnsupportedOperationException) { + dbg = true; + } + try { + if (dbg) { + log.debug("create index and update tap_schema failed - rollback", ex); + } else { + log.error("create index and update tap_schema failed - rollback", ex); + } + tm.rollbackTransaction(); + if (dbg) { + log.debug("create index and update tap_schema failed - rollback: OK"); + } else { + log.error("create index and update tap_schema failed - rollback: OK"); + } + } catch (Exception oops) { + log.error("create index and update tap_schema - rollback : FAIL", oops); + } + if (ex instanceof IllegalArgumentException) { + throw ex; + } + throw new RuntimeException("failed to update table " + tableName + " reason: " + ex.getMessage(), ex); + } finally { + if (tm.isOpen()) { + log.error("BUG: open transaction in finally - trying to rollback"); + try { + tm.rollbackTransaction(); + log.error("BUG: rollback in finally: OK"); + } catch (Exception oops) { + log.error("BUG: rollback in finally: FAIL", oops); + } + throw new RuntimeException("BUG: open transaction in finally"); + } + } + } + + /** + * + * @param params + */ + protected void ingestTable(Map> params) { + String tableName = getSingleValue("table", params); + if (tableName == null) { + throw new IllegalArgumentException("missing parameter 'table'"); + } + + // check write permissions to the tap_schema + PluginFactory pf = new PluginFactory(); + TapSchemaDAO ts = pf.getTapSchemaDAO(); + DataSource ds = getDataSource(); + ts.setDataSource(ds); + + + try { + TablesAction.checkSchemaWritePermissions(ts, "tap_schema", logInfo); + } catch (ResourceNotFoundException | IOException ex) { + throw new RuntimeException("tap_schema not found"); + } + + // check if table already exists in tap_schema + TableDesc tableDesc = ts.getTable(tableName); + if (tableDesc != null) { + throw new IllegalArgumentException("ingest table already exists in tap_schema: " + tableName); + } + + // add the table to the tap_schema + TableIngester tableIngester = new TableIngester(ds); + tableIngester.ingest("tap_schema", tableName); + } + private String getSingleValue(String pname, Map> params) { List vals = params.get(pname); if (vals == null || vals.isEmpty()) { diff --git a/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java new file mode 100644 index 00000000..14e1516a --- /dev/null +++ b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java @@ -0,0 +1,201 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package ca.nrc.cadc.tap.pg; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.tap.db.TableCreator; +import ca.nrc.cadc.tap.db.TableIngester; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import ca.nrc.cadc.util.Log4jInit; +import java.util.List; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +public class TableIngesterTest { + private static final Logger log = Logger.getLogger(TableIngesterTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.tap.db", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.tap.schema", Level.DEBUG); + } + + private final DataSource dataSource; + private final TapSchemaDAO tapSchemaDAO; + private final String TEST_SCHEMA = "tap_schema"; + + public TableIngesterTest() { + // create a datasource and register with JNDI + try { + DBConfig conf = new DBConfig(); + ConnectionConfig cc = conf.getConnectionConfig("TAP_SCHEMA_TEST", "cadctest"); + this.dataSource = DBUtil.getDataSource(cc); + log.info("configured data source: " + cc.getServer() + "," + cc.getDatabase() + "," + cc.getDriver() + "," + cc.getURL()); + this.tapSchemaDAO = new TapSchemaDAO(); + this.tapSchemaDAO.setDataSource(this.dataSource); + } catch (Exception ex) { + log.error("setup failed", ex); + throw new IllegalStateException("failed to create DataSource", ex); + } + } + + @Test + public void testTableIngest() { + String testTable = TEST_SCHEMA + ".testTableIngest"; + TableCreator tableCreator = new TableCreator(dataSource); + try { + // cleanup + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + // ingest table into the tap_schema + TableIngester tableIngester = new TableIngester(dataSource); + tableIngester.ingest(TEST_SCHEMA, testTable); + log.debug("ingested table"); + + // compare database and tap_schema + SchemaDesc schemaDesc = tapSchemaDAO.getSchema(TEST_SCHEMA, true); + Assert.assertNotNull("schema", schemaDesc); + + TableDesc tableDesc = tapSchemaDAO.getTable(testTable); + Assert.assertNotNull("table", tableDesc); + + List databaseColumns = ingestTable.getColumnDescs(); + List tapSchemaColumns = tableDesc.getColumnDescs(); + for (ColumnDesc databaseColumn: databaseColumns) { + boolean found = false; + log.debug("database column: " + databaseColumn.getColumnName()); + for (ColumnDesc tapSchemaColumn : tapSchemaColumns) { + log.debug("tap_schema column: " + tapSchemaColumn.getColumnName()); + if (databaseColumn.getColumnName().equals(tapSchemaColumn.getColumnName())) { + Assert.assertEquals("datatype", databaseColumn.getDatatype(), tapSchemaColumn.getDatatype()); + found = true; + break; + } + } + Assert.assertTrue("tap_schema column not found: " + databaseColumn, found); + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + TableDesc getTableDesc(String schemaName, String tableName) throws Exception { + final TableDesc tableDesc = new TableDesc(schemaName, tableName); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e7", TapDataType.INTERVAL)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e8", TapDataType.POINT)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e9", TapDataType.CIRCLE)); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e10", TapDataType.POLYGON)); + // arrays + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a11", new TapDataType("short", "*", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a12", new TapDataType("int", "*", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a13", new TapDataType("long", "*", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a14", new TapDataType("float", "*", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a15", new TapDataType("double", "*", null))); + + return tableDesc; + } + +} From 2469ac2bb410bb080b575320640aebd426b7a9ac Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 4 Sep 2024 13:49:22 -0700 Subject: [PATCH 04/14] CADC-13452 check the caller has write permission on the schema before ingesting a table --- .../cadc/vosi/actions/TableUpdateRunner.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java index 743b5fba..b0d089d6 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java @@ -225,8 +225,8 @@ private void doit() { } /** - * - * @param params + * Add a index to a column in the tap_schema and create the index in the database. + * @param params list of request query parameters. */ protected void indexTable(Map> params) { String tableName = getSingleValue("table", params); @@ -327,8 +327,9 @@ protected void indexTable(Map> params) { } /** + * Add the metadata for an existing table to the tap_schema. * - * @param params + * @param params list of request query parameters. */ protected void ingestTable(Map> params) { String tableName = getSingleValue("table", params); @@ -336,17 +337,17 @@ protected void ingestTable(Map> params) { throw new IllegalArgumentException("missing parameter 'table'"); } - // check write permissions to the tap_schema PluginFactory pf = new PluginFactory(); TapSchemaDAO ts = pf.getTapSchemaDAO(); DataSource ds = getDataSource(); ts.setDataSource(ds); - + // check write permissions to the tap_schema + String schemaName = Util.getSchemaFromTable(tableName); try { - TablesAction.checkSchemaWritePermissions(ts, "tap_schema", logInfo); + TablesAction.checkSchemaWritePermissions(ts, schemaName, logInfo); } catch (ResourceNotFoundException | IOException ex) { - throw new RuntimeException("tap_schema not found"); + throw new IllegalArgumentException("ingest schema not found in tap_schema: " + schemaName); } // check if table already exists in tap_schema @@ -357,7 +358,7 @@ protected void ingestTable(Map> params) { // add the table to the tap_schema TableIngester tableIngester = new TableIngester(ds); - tableIngester.ingest("tap_schema", tableName); + tableIngester.ingest(schemaName, tableName); } private String getSingleValue(String pname, Map> params) { From a4a81239bae3ffa9566f032f9bb064e08e9fdfdb Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 5 Sep 2024 07:47:00 -0700 Subject: [PATCH 05/14] CADC-13452 copy int-tests from youcat to cadc-tap-schema --- .../ca/nrc/cadc/tap/db/TableIngesterTest.java | 167 +++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java index 6ec6c899..61d74e7c 100644 --- a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java +++ b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java @@ -78,13 +78,20 @@ import ca.nrc.cadc.tap.schema.TapDataType; import ca.nrc.cadc.tap.schema.TapSchemaDAO; import ca.nrc.cadc.util.Log4jInit; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import javax.sql.DataSource; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; public class TableIngesterTest { private static final Logger log = Logger.getLogger(TableIngesterTest.class); @@ -116,9 +123,9 @@ public TableIngesterTest() { @Test public void testTableIngest() { String testTable = TEST_SCHEMA + ".testTableIngest"; - TableCreator tableCreator = new TableCreator(dataSource); try { // cleanup + TableCreator tableCreator = new TableCreator(dataSource); try { tableCreator.dropTable(testTable); } catch (Exception ignore) { @@ -167,13 +174,13 @@ public void testTableIngest() { // cleanup try { -// tableCreator.dropTable(testTable); + tableCreator.dropTable(testTable); log.debug("dropped table: " + testTable); } catch (Exception ignore) { log.debug("database-cleanup-after-test failed for " + testTable); } try { -// tapSchemaDAO.delete(testTable); + tapSchemaDAO.delete(testTable); } catch (Exception ignore) { log.debug("tap_schema-cleanup-after-test failed for " + testTable); } @@ -183,6 +190,160 @@ public void testTableIngest() { } } + @Test + public void testVerifyTableMetadata() { + String testTableName = "testVerifyTableMetadata"; + String testTable = TEST_SCHEMA + "." + testTableName; + + try { + // cleanup + TableCreator tableCreator = new TableCreator(dataSource); + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + try (Connection connection = dataSource.getConnection()) { + DatabaseMetaData metaData = connection.getMetaData(); + try (ResultSet rs = metaData.getColumns(null, TEST_SCHEMA, testTableName.toLowerCase(), null)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + switch (colName) { + case "c0": + Assert.assertEquals("String", "varchar", rs.getString("TYPE_NAME")); + break; + case "c1": + Assert.assertEquals("short", "int2", rs.getString("TYPE_NAME")); + break; + case "c2": + Assert.assertEquals("integer", "int4", rs.getString("TYPE_NAME")); + break; + case "c3": + Assert.assertEquals("long", "int8", rs.getString("TYPE_NAME")); + break; + case "c4": + Assert.assertEquals("float", "float4", rs.getString("TYPE_NAME")); + break; + case "c5": + Assert.assertEquals("double", "float8", rs.getString("TYPE_NAME")); + break; + case "c6": + Assert.assertEquals("timestamp", "timestamp", rs.getString("TYPE_NAME")); + break; + default: + Assert.fail("unexpected column: " + colName); + } + } + } + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Ignore + @Test + public void testPrintTableMetadata() { + String testTableName = "testPrintTableMetadata"; + String testTable = TEST_SCHEMA + "." + testTableName; + try { + // cleanup + TableCreator tableCreator = new TableCreator(dataSource); + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + List columnInfo = new ArrayList() {{ + add("DATA_TYPE"); + add("TYPE_NAME"); + add("COLUMN_SIZE"); + add("COLUMN_DEF"); + add("DECIMAL_DIGITS"); + add("SQL_DATA_TYPE"); + add("SQL_DATETIME_SUB"); + add("CHAR_OCTET_LENGTH"); + add("SOURCE_DATA_TYPE"); + add("REMARKS"); + add("IS_NULLABLE"); + }}; + + List indexInfo = new ArrayList() {{ + add("NON_UNIQUE"); + add("INDEX_QUALIFIER"); + add("INDEX_NAME"); + add("TYPE"); + add("ASC_OR_DESC"); + }}; + + try (Connection connection = dataSource.getConnection()) { + // create two indexes on the table + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + String index1 = "CREATE UNIQUE INDEX c0_idx ON int_test_schema.testPrintTableMetadata (c0)"; + log.debug("sql:\n" + index1); + jdbc.execute(index1); + String index2 = "CREATE UNIQUE INDEX c6_idx ON int_test_schema.testPrintTableMetadata (c6)"; + log.debug("sql:\n" + index2); + jdbc.execute(index2); + + log.info("column metadata"); + DatabaseMetaData metaData = connection.getMetaData(); + try (ResultSet rs = metaData.getColumns(null, TEST_SCHEMA, testTableName.toLowerCase(), null)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + log.info(String.format("columnName: %s", colName)); + for (String info : columnInfo) { + log.info(String.format("%s.%s: %s", colName, info, rs.getString(info))); + } + } + } + + log.info("index metadata"); + try (ResultSet rs = metaData.getIndexInfo(null, TEST_SCHEMA, testTableName.toLowerCase(), false, false)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + log.info(String.format("columnName: %s", colName)); + for (String info : indexInfo) { + log.info(String.format("%s.%s: %s", colName, info, rs.getString(info))); + } + } + } + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + TableDesc getTableDesc(String schemaName, String tableName) throws Exception { final TableDesc tableDesc = new TableDesc(schemaName, tableName); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "c0", TapDataType.STRING)); From 30cbff8fb30a4acdd542e1172c91ffdfd577ba07 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 5 Sep 2024 09:00:27 -0700 Subject: [PATCH 06/14] CADC-13452 add TableIngesterTest to cadc-tap-server-pg --- .../ca/nrc/cadc/tap/pg/TableIngesterTest.java | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java index 14e1516a..5c4c54e4 100644 --- a/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java +++ b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java @@ -80,12 +80,18 @@ import ca.nrc.cadc.tap.schema.TapDataType; import ca.nrc.cadc.tap.schema.TapSchemaDAO; import ca.nrc.cadc.util.Log4jInit; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.ArrayList; import java.util.List; import javax.sql.DataSource; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; public class TableIngesterTest { private static final Logger log = Logger.getLogger(TableIngesterTest.class); @@ -182,6 +188,166 @@ public void testTableIngest() { } } + @Test + public void testVerifyTableMetadata() { + String testTableName = "testVerifyTableMetadata"; + String testTable = TEST_SCHEMA + "." + testTableName; + + try { + // cleanup + TableCreator tableCreator = new TableCreator(dataSource); + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + try (Connection connection = dataSource.getConnection()) { + DatabaseMetaData metaData = connection.getMetaData(); + try (ResultSet rs = metaData.getColumns(null, TEST_SCHEMA, testTableName.toLowerCase(), null)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + switch (colName) { + case "e7": + Assert.assertEquals("interval", "polygon", rs.getString("TYPE_NAME")); + break; + case "e8": + Assert.assertEquals("point", "spoint", rs.getString("TYPE_NAME")); + break; + case "e9": + Assert.assertEquals("circle", "scircle", rs.getString("TYPE_NAME")); + break; + case "e10": + Assert.assertEquals("polygon", "spoly", rs.getString("TYPE_NAME")); + break; + case "a11": + Assert.assertEquals("short[]", "_int2", rs.getString("TYPE_NAME")); + break; + case "a12": + Assert.assertEquals("int[]", "_int4", rs.getString("TYPE_NAME")); + break; + case "a13": + Assert.assertEquals("long[]", "_int8", rs.getString("TYPE_NAME")); + break; + case "a14": + Assert.assertEquals("float[]", "_float4", rs.getString("TYPE_NAME")); + break; + case "a15": + Assert.assertEquals("double[]", "_float8", rs.getString("TYPE_NAME")); + break; + default: + Assert.fail("unexpected column: " + colName); + } + } + } + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Ignore + @Test + public void testPrintTableMetadata() { + String testTableName = "testPrintTableMetadata"; + String testTable = TEST_SCHEMA + "." + testTableName; + try { + // cleanup + TableCreator tableCreator = new TableCreator(dataSource); + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + + // create test table in the database + TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); + tableCreator.createTable(ingestTable); + log.debug("created database table: " + testTable); + + List columnInfo = new ArrayList() {{ + add("DATA_TYPE"); + add("TYPE_NAME"); + add("COLUMN_SIZE"); + add("COLUMN_DEF"); + add("DECIMAL_DIGITS"); + add("SQL_DATA_TYPE"); + add("SQL_DATETIME_SUB"); + add("CHAR_OCTET_LENGTH"); + add("SOURCE_DATA_TYPE"); + add("REMARKS"); + add("IS_NULLABLE"); + }}; + + List indexInfo = new ArrayList() {{ + add("NON_UNIQUE"); + add("INDEX_QUALIFIER"); + add("INDEX_NAME"); + add("TYPE"); + add("ASC_OR_DESC"); + }}; + + try (Connection connection = dataSource.getConnection()) { + // create two indexes on the table + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + String index1 = "CREATE UNIQUE INDEX c0_idx ON int_test_schema.testPrintTableMetadata (c0)"; + log.debug("sql:\n" + index1); + jdbc.execute(index1); + String index2 = "CREATE UNIQUE INDEX c6_idx ON int_test_schema.testPrintTableMetadata (c6)"; + log.debug("sql:\n" + index2); + jdbc.execute(index2); + + log.info("column metadata"); + DatabaseMetaData metaData = connection.getMetaData(); + try (ResultSet rs = metaData.getColumns(null, TEST_SCHEMA, testTableName.toLowerCase(), null)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + log.info(String.format("columnName: %s", colName)); + for (String info : columnInfo) { + log.info(String.format("%s.%s: %s", colName, info, rs.getString(info))); + } + } + } + + log.info("index metadata"); + try (ResultSet rs = metaData.getIndexInfo(null, TEST_SCHEMA, testTableName.toLowerCase(), false, false)) { + while (rs.next()) { + String colName = rs.getString("COLUMN_NAME"); + log.info(String.format("columnName: %s", colName)); + for (String info : indexInfo) { + log.info(String.format("%s.%s: %s", colName, info, rs.getString(info))); + } + } + } + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + TableDesc getTableDesc(String schemaName, String tableName) throws Exception { final TableDesc tableDesc = new TableDesc(schemaName, tableName); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e7", TapDataType.INTERVAL)); From cf3665f928aa0da2303d5776ae738494f8fd3a86 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 11 Sep 2024 13:17:11 -0700 Subject: [PATCH 07/14] CADC-13452 add int-test for an unsupported datatype in a table ingest --- .../ca/nrc/cadc/tap/db/TableIngesterTest.java | 69 ++++++++++++++++++- .../ca/nrc/cadc/tap/db/TableIngester.java | 20 ++++-- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java index 61d74e7c..86dc19bc 100644 --- a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java +++ b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java @@ -122,7 +122,7 @@ public TableIngesterTest() { @Test public void testTableIngest() { - String testTable = TEST_SCHEMA + ".testTableIngest"; + String testTable = TEST_SCHEMA + ".test_table_ingest"; try { // cleanup TableCreator tableCreator = new TableCreator(dataSource); @@ -256,6 +256,73 @@ public void testVerifyTableMetadata() { } } + // test only runs using a PostgreSQL database because + // it is using a pg specific unsupported datatype. + @Test + public void testUnsupportedDataType() throws Exception { + + DatabaseMetaData databaseMetaData = dataSource.getConnection().getMetaData(); + String dbProductName = databaseMetaData.getDatabaseProductName(); + if (!"PostgreSQL".equals(dbProductName)) { + log.info("expected PostgreSQL database, found unsupported database: " + dbProductName); + return; + } + + String testTable = TEST_SCHEMA + ".test_unsupported_datatype"; + try { + // cleanup + TableCreator tableCreator = new TableCreator(dataSource); + try { + tableCreator.dropTable(testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-before-test failed for " + testTable); + } + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-before-test failed for " + testTable); + } + + // create test table in the database with an unsupported data type + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + String unsupportedDataType = "money"; + String sql = String.format("CREATE TABLE %s (c1 varchar(16), i1 integer, m1 %s)", + testTable, unsupportedDataType); + log.debug("sql:\n" + sql); + jdbc.execute(sql); + log.debug("created database table: " + testTable); + + // ingest table into the tap_schema checking for expected exception + boolean success = false; + try { + TableIngester tableIngester = new TableIngester(dataSource); + tableIngester.ingest(TEST_SCHEMA, testTable); + } catch (UnsupportedOperationException expected) { + log.info("expected exception: " + expected); + success = true; + } + + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-after-test failed for " + testTable); + } + + if (!success) { + Assert.fail("unsupported data type should throw an exception"); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } @Ignore @Test public void testPrintTableMetadata() { diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java index f67f8f88..152f6eb1 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java @@ -92,8 +92,6 @@ public class TableIngester { private static final Logger log = Logger.getLogger(TableIngester.class); - static final String TAP_SCHEMA = "tap_schema"; - private final DataSource dataSource; private final DatabaseDataType databaseDataType; private final TapSchemaDAO tapSchemaDAO; @@ -188,10 +186,12 @@ protected TableDesc createTableDesc(String schemaName, String tableName) throws SQLException { log.debug(String.format("creating TableDesc for %s %s", schemaName, tableName)); // get the table metadata + String unqualifiedTableName = getUnqualifiedTableNameFromTable(tableName); DatabaseMetaData databaseMetaData = dataSource.getConnection().getMetaData(); + log.debug(String.format("querying DatabaseMetadata for schema=%s table=%s", schemaName, unqualifiedTableName)); //TODO too pg specific? table names are stored lower case in the system tables queried for the metadata - ResultSet columnInfo = databaseMetaData.getColumns(null, schemaName, "testtableingest", null); - ResultSet indexInfo = databaseMetaData.getIndexInfo(null, schemaName, "testtableingest", false, false); + ResultSet columnInfo = databaseMetaData.getColumns(null, schemaName, unqualifiedTableName.toLowerCase(), null); + ResultSet indexInfo = databaseMetaData.getIndexInfo(null, schemaName, unqualifiedTableName.toLowerCase(), false, false); // get column names for indexed columns List indexedColumns = new ArrayList(); while (indexInfo.next()) { @@ -201,9 +201,9 @@ protected TableDesc createTableDesc(String schemaName, String tableName) } // build TableDesc - TableDesc tableDesc = new TableDesc(TAP_SCHEMA, tableName); + TableDesc tableDesc = new TableDesc(schemaName, tableName); tableDesc.tableType = TableDesc.TableType.TABLE; - log.debug(String.format("creating TableDesc %s %s", TAP_SCHEMA, tableName)); + log.debug(String.format("creating TableDesc %s %s", schemaName, tableName)); while (columnInfo.next()) { String columnName = columnInfo.getString("COLUMN_NAME"); String columnType = columnInfo.getString("TYPE_NAME"); @@ -216,4 +216,12 @@ protected TableDesc createTableDesc(String schemaName, String tableName) return tableDesc; } + String getUnqualifiedTableNameFromTable(String tableName) { + String[] st = tableName.split("[.]"); + if (st.length == 2) { + return st[1]; + } + throw new IllegalArgumentException("invalid table name: " + tableName + " (expected: .)"); + } + } From 185f6c790a39fb9487ccfe8e03b1a81ef06bc8c9 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 11 Sep 2024 13:18:31 -0700 Subject: [PATCH 08/14] CADC-13452 update TableUpdateRunner for updated table-update API --- .../cadc/vosi/actions/TableUpdateRunner.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java index b0d089d6..df1bc2dc 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java @@ -120,10 +120,10 @@ public class TableUpdateRunner implements JobRunner { private static final List PARAM_NAMES = new ArrayList(); static { - PARAM_NAMES.add("table"); PARAM_NAMES.add("index"); + PARAM_NAMES.add("ingest"); + PARAM_NAMES.add("table"); PARAM_NAMES.add("unique"); - PARAM_NAMES.add("op"); } private JobUpdater jobUpdater; @@ -178,18 +178,17 @@ private void doit() { // check for the requested operation ParamExtractor pe = new ParamExtractor(PARAM_NAMES); Map> params = pe.getParameters(job.getParameterList()); - String op = getSingleValue("op", params); - if (op == null) { - throw new IllegalArgumentException("missing parameter 'op'"); + String index = getSingleValue("index", params); + String ingest = getSingleValue("ingest", params); + if (index == null && ingest == null) { + throw new IllegalArgumentException("one of 'index' or 'ingest' parameter must be specified"); + } else if (index != null && ingest != null) { + throw new IllegalArgumentException("'index' and 'ingest' parameters cannot be specified at the same time"); } - - // index or ingest table - if (op.equals("index")) { + if (index != null) { indexTable(params); - } else if (op.equals("ingest")) { - ingestTable(params); } else { - throw new IllegalArgumentException("unknown 'op' parameter: " + op); + ingestTable(params); } ep = jobUpdater.setPhase(job.getID(), ExecutionPhase.EXECUTING, ExecutionPhase.COMPLETED, new Date()); @@ -230,15 +229,12 @@ private void doit() { */ protected void indexTable(Map> params) { String tableName = getSingleValue("table", params); - String columnName = getSingleValue("column", params); + String columnName = getSingleValue("index", params); boolean unique = "true".equals(getSingleValue("unique", params)); if (tableName == null) { throw new IllegalArgumentException("missing parameter 'table'"); } - if (columnName == null) { - throw new IllegalArgumentException("missing parameter 'column'"); - } PluginFactory pf = new PluginFactory(); TapSchemaDAO ts = pf.getTapSchemaDAO(); @@ -332,10 +328,16 @@ protected void indexTable(Map> params) { * @param params list of request query parameters. */ protected void ingestTable(Map> params) { + boolean ingest = "true".equals(getSingleValue("ingest", params)); + if (!ingest) { + throw new IllegalStateException("'ingest' parameter specified but value is 'false', ingest cancelled"); + } + String tableName = getSingleValue("table", params); if (tableName == null) { throw new IllegalArgumentException("missing parameter 'table'"); } + log.debug("ingesting table " + tableName); PluginFactory pf = new PluginFactory(); TapSchemaDAO ts = pf.getTapSchemaDAO(); From a608ca5050a90f93e553720dccb8f56aeac16d9f Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Wed, 11 Sep 2024 13:31:42 -0700 Subject: [PATCH 09/14] CADC-13452 add TableUpdateTest for table-update endpoint, move in table index tests from CreateTableTest --- youcat/build.gradle | 3 +- .../opencadc/youcat/AbstractTablesTest.java | 2 +- .../org/opencadc/youcat/CreateTableTest.java | 56 +--- .../org/opencadc/youcat/TableUpdateTest.java | 303 ++++++++++++++++++ 4 files changed, 307 insertions(+), 57 deletions(-) create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java diff --git a/youcat/build.gradle b/youcat/build.gradle index 4a2eb6ed..dd2c1d9a 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -51,12 +51,13 @@ dependencies { intTestCompile 'org.opencadc:cadc-test-uws:[1.1.1,)' intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' intTestCompile 'org.opencadc:cadc-test-tap:[1.1,)' + + intTestRuntime 'org.postgresql:postgresql:[42.2,43.0)' } configurations { // this are provided by tomcat runtime.exclude group: 'javax.servlet' - runtime.exclude group: 'org.postgresql' // pulled by stil,unused, critical CVEs: runtime.exclude group: 'org.yaml', module: 'snakeyaml' diff --git a/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java index cef78227..8cc3f933 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java @@ -252,8 +252,8 @@ void doCreateIndex(Subject subject, String tableName, String indexCol, boolean u // create job Map params = new TreeMap(); - params.put("table", tableName); params.put("index", indexCol); + params.put("table", tableName); params.put("unique", Boolean.toString(unique)); HttpPost post = new HttpPost(certUpdateURL, params, false); Subject.doAs(subject, new RunnableAction(post)); diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java index 2d4e0f68..5a6418b0 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java @@ -84,7 +84,6 @@ import ca.nrc.cadc.tap.schema.TapDataType; import ca.nrc.cadc.tap.schema.TapPermissions; import ca.nrc.cadc.util.Log4jInit; -import ca.nrc.cadc.uws.ExecutionPhase; import ca.nrc.cadc.vosi.InvalidTableSetException; import ca.nrc.cadc.vosi.TableReader; import ca.nrc.cadc.vosi.actions.TableDescHandler; @@ -276,60 +275,7 @@ public void write(OutputStream out) throws IOException { Assert.fail("unexpected exception: " + unexpected); } } - - @Test - public void testCreateIndex() { - try { - clearSchemaPerms(); - TapPermissions tp = new TapPermissions(null, true, null, null); - super.setPerms(schemaOwner, testSchemaName, tp, 200); - - String tableName = testSchemaName + ".testCreateIndex"; - TableDesc td = doCreateTable(schemaOwner, tableName); - for (ColumnDesc cd : td.getColumnDescs()) { - log.info("testCreateIndex: " + cd.getColumnName()); - ExecutionPhase expected = ExecutionPhase.COMPLETED; - if (cd.getColumnName().startsWith("a")) { - expected = ExecutionPhase.ERROR; - } - doCreateIndex(schemaOwner, tableName, cd.getColumnName(), false,expected, null); - } - - // cleanup on success - doDelete(schemaOwner, td.getTableName(), false); - } catch (Exception unexpected) { - log.error("unexpected exception", unexpected); - Assert.fail("unexpected exception: " + unexpected); - } - } - - @Test - public void testCreateUniqueIndex() { - try { - clearSchemaPerms(); - TapPermissions tp = new TapPermissions(null, true, null, null); - super.setPerms(schemaOwner, testSchemaName, tp, 200); - - String tableName = testSchemaName + ".testCreateUniqueIndex"; - TableDesc td = doCreateTable(schemaOwner, tableName); - for (ColumnDesc cd : td.getColumnDescs()) { - - ExecutionPhase expected = ExecutionPhase.COMPLETED; - if (cd.getColumnName().startsWith("e") || cd.getColumnName().startsWith("a")) { - expected = ExecutionPhase.ERROR; // unique index not allowed - } - log.info("testCreateUniqueIndex: " + cd.getColumnName() + " expect: " + expected.getValue()); - doCreateIndex(schemaOwner, tableName, cd.getColumnName(), true, expected, null); - } - - // cleanup on success - doDelete(schemaOwner, tableName, false); - } catch (Exception unexpected) { - log.error("unexpected exception", unexpected); - Assert.fail("unexpected exception: " + unexpected); - } - } - + private void compare(TableDesc expected, TableDesc actual) { // When you read just a single table document you do not get the schema name and TableReader makes one up //Assert.assertEquals("schema name", "default", actual.getSchemaName()); diff --git a/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java new file mode 100644 index 00000000..8ce31352 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java @@ -0,0 +1,303 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2024. (c) 2024. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; +import ca.nrc.cadc.net.OutputStreamWrapper; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.uws.ExecutionPhase; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.JobReader; +import ca.nrc.cadc.vosi.TableWriter; +import ca.nrc.cadc.vosi.actions.TableDescHandler; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.URL; +import java.sql.DatabaseMetaData; +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; +import javax.sql.DataSource; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.jdbc.core.JdbcTemplate; + +public class TableUpdateTest extends AbstractTablesTest { + private static final Logger log = Logger.getLogger(TableUpdateTest.class); + + static { + Log4jInit.setLevel("org.opencadc.youcat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + } + + private final DataSource dataSource; + private final TapSchemaDAO tapSchemaDAO; + + public TableUpdateTest() { + super(); + + try { + DBConfig conf = new DBConfig(); + ConnectionConfig cc = conf.getConnectionConfig("YOUCAT_TEST", "cadctest"); + this.dataSource = DBUtil.getDataSource(cc); + log.info("configured data source: " + cc.getServer() + "," + cc.getDatabase() + "," + cc.getDriver() + "," + cc.getURL()); + this.tapSchemaDAO = new TapSchemaDAO(); + this.tapSchemaDAO.setDataSource(this.dataSource); + } catch (Exception ex) { + log.error("setup failed", ex); + throw new IllegalStateException("failed to create DataSource", ex); + } + } + + @Test + public void testCreateIndex() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, testSchemaName, tp, 200); + + String tableName = testSchemaName + ".testCreateIndex"; + TableDesc td = doCreateTable(schemaOwner, tableName); + for (ColumnDesc cd : td.getColumnDescs()) { + log.info("testCreateIndex: " + cd.getColumnName()); + ExecutionPhase expected = ExecutionPhase.COMPLETED; + if (cd.getColumnName().startsWith("a")) { + expected = ExecutionPhase.ERROR; + } + doCreateIndex(schemaOwner, tableName, cd.getColumnName(), false,expected, null); + } + + // cleanup on success + doDelete(schemaOwner, td.getTableName(), false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testCreateUniqueIndex() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, testSchemaName, tp, 200); + + String tableName = testSchemaName + ".testCreateUniqueIndex"; + TableDesc td = doCreateTable(schemaOwner, tableName); + for (ColumnDesc cd : td.getColumnDescs()) { + + ExecutionPhase expected = ExecutionPhase.COMPLETED; + if (cd.getColumnName().startsWith("e") || cd.getColumnName().startsWith("a")) { + expected = ExecutionPhase.ERROR; // unique index not allowed + } + log.info("testCreateUniqueIndex: " + cd.getColumnName() + " expect: " + expected.getValue()); + doCreateIndex(schemaOwner, tableName, cd.getColumnName(), true, expected, null); + } + + // cleanup on success + doDelete(schemaOwner, tableName, false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testIngestTest() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, testSchemaName, tp, 200); + + // create test table and schema + String testTable = testSchemaName + ".test_ingest_table"; + TableDesc td = doCreateTable(schemaOwner, testTable); + + // delete the schema from tap_schema + try { + tapSchemaDAO.delete(testTable); + } catch (Exception ignore) { + log.debug("tap_schema-cleanup-before-test failed for " + testTable); + } + + // run the ingest + doIngestTable(schemaOwner, testTable); + + // cleanup on success + doDelete(schemaOwner, td.getTableName(), false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + // test only runs using a PostgreSQL database because + // it is using a pg specific unsupported datatype. + @Test + public void testIngestUnsupportedDataType() { + try { + DatabaseMetaData databaseMetaData = dataSource.getConnection().getMetaData(); + String dbProductName = databaseMetaData.getDatabaseProductName(); + if (!"PostgreSQL".equals(dbProductName)) { + log.info("expected PostgreSQL database, found unsupported database: " + dbProductName); + return; + } + + // create test table in the database with an unsupported data type + String testTable = testSchemaName + ".test_unsupported_datatype"; + String unsupportedDataType = "money"; + JdbcTemplate jdbc = new JdbcTemplate(dataSource); + String sql = String.format("CREATE TABLE %s (c1 varchar(16), i1 integer, m1 %s)", + testTable, unsupportedDataType); + log.debug("sql:\n" + sql); + try { + jdbc.execute(sql); + } catch (Exception e) { + Assert.fail(String.format("error creating table %s because %s", testTable, e.getMessage())); + } + log.debug("created database table: " + testTable); + + try { + doIngestTable(schemaOwner, testTable); + } catch (UnsupportedOperationException expected) { + log.info("expected exception: " + expected); + } + + // cleanup on success + doDelete(schemaOwner, testTable, true); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + void doIngestTable(Subject subject, String tableName) throws Exception { + // create job + Map params = new TreeMap(); + params.put("op", "ingest"); + params.put("table", tableName); + HttpPost post = new HttpPost(certUpdateURL, params, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertNull("throwable", post.getThrowable()); + Assert.assertEquals("response code", 303, post.getResponseCode()); + final URL jobURL = post.getRedirectURL(); + Assert.assertNotNull("jobURL", jobURL); + log.info("ingest table job: " + jobURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(jobURL, bos); + Subject.doAs(subject, new RunnableAction(get)); + Assert.assertNull("throwable", get.getThrowable()); + Assert.assertEquals("response code", 200, get.getResponseCode()); + String xml = bos.toString(); + log.debug("ingest table job:\n" + xml); + + // execute job + params.clear(); + params.put("PHASE", "RUN"); + final URL phaseURL = new URL(jobURL.toExternalForm() + "/phase"); + post = new HttpPost(phaseURL, params, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertNull("throwable", post.getThrowable()); + + // wait for completion + bos.reset(); + final URL blockURL = new URL(jobURL.toExternalForm() + "?WAIT=60"); + HttpGet block = new HttpGet(blockURL, bos); + Subject.doAs(subject, new RunnableAction(block)); + Assert.assertNull("throwable", block.getThrowable()); + Assert.assertEquals("response code", 200, block.getResponseCode()); + String xml2 = bos.toString(); + log.debug("final job state:\n" + xml2); + + JobReader r = new JobReader(); + Job end = r.read(new StringReader(xml2)); + log.debug(end.getErrorSummary().toString()); +// Assert.assertEquals("final job state", ExecutionPhase.COMPLETED, end.getExecutionPhase()); + } + +} From 40959f372adb221f0364d829296235d8e0f1c11a Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 12 Sep 2024 08:47:49 -0700 Subject: [PATCH 10/14] CADC-13452 update build jar versions --- cadc-tap-server-pg/build.gradle | 2 +- youcat/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cadc-tap-server-pg/build.gradle b/cadc-tap-server-pg/build.gradle index 48ede01f..d02355fa 100644 --- a/cadc-tap-server-pg/build.gradle +++ b/cadc-tap-server-pg/build.gradle @@ -24,7 +24,7 @@ def git_url = 'https://github.com/opencadc/tap' dependencies { compile 'org.opencadc:cadc-dali-pg:[0.1,)' compile 'org.opencadc:cadc-adql:[1.1.14,)' - compile 'org.opencadc:cadc-tap-schema:[1.1.34, )' + compile 'org.opencadc:cadc-tap-schema:[1.2.1, )' compile 'org.opencadc:cadc-tap-server:[1.1.7, )' testCompile 'junit:junit:[4.0,5.0)' diff --git a/youcat/build.gradle b/youcat/build.gradle index dd2c1d9a..8cee67fb 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -34,9 +34,9 @@ dependencies { compile 'org.opencadc:cadc-adql:[1.1.14,)' compile 'org.opencadc:cadc-uws:[1.0.2,)' compile 'org.opencadc:cadc-uws-server:[1.2.22,)' - compile 'org.opencadc:cadc-tap-schema:[1.1.32,)' + compile 'org.opencadc:cadc-tap-schema:[1.2.1,)' compile 'org.opencadc:cadc-tap-server:[1.1.24,)' - compile 'org.opencadc:cadc-tap-server-pg:[1.1.0,)' + compile 'org.opencadc:cadc-tap-server-pg:[1.1.1,)' compile 'org.opencadc:cadc-adql:[1.1.4,)' compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' compile 'org.opencadc:cadc-registry:[1.7.2,)' From aa2a9164e55c659c858806baa31973f58754c8fa Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 12 Sep 2024 08:48:33 -0700 Subject: [PATCH 11/14] CADC-13452 various fixes --- .../ca/nrc/cadc/vosi/actions/PutAction.java | 2 +- .../cadc/vosi/actions/TableUpdateRunner.java | 1 + .../org/opencadc/youcat/TableUpdateTest.java | 18 ++++++++++-------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java index 14f0fecf..e83c9485 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java @@ -219,7 +219,7 @@ private void createTable(TapSchemaDAO ts, String schemaName, String tableName) t throw new ResourceAlreadyExistsException("table " + tableName + " already exists"); } // flag table as created using the API to allow table deletion in the DeleteAction - td.apiCreated = true; + inputTable.apiCreated = true; Profiler prof = new Profiler(PutAction.class); DatabaseTransactionManager tm = new DatabaseTransactionManager(ds); diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java index df1bc2dc..688afce8 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TableUpdateRunner.java @@ -231,6 +231,7 @@ protected void indexTable(Map> params) { String tableName = getSingleValue("table", params); String columnName = getSingleValue("index", params); boolean unique = "true".equals(getSingleValue("unique", params)); + log.debug(String.format("indexing table=%s column=%s unique=%s", tableName, columnName, unique)); if (tableName == null) { throw new IllegalArgumentException("missing parameter 'table'"); diff --git a/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java index 8ce31352..8f3f778f 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java @@ -188,14 +188,17 @@ public void testCreateUniqueIndex() { } @Test - public void testIngestTest() { + public void testIngestTable() { try { clearSchemaPerms(); TapPermissions tp = new TapPermissions(null, true, null, null); super.setPerms(schemaOwner, testSchemaName, tp, 200); - // create test table and schema + // cleanup String testTable = testSchemaName + ".test_ingest_table"; + doDelete(schemaOwner, testTable, true); + + // create test table and schema TableDesc td = doCreateTable(schemaOwner, testTable); // delete the schema from tap_schema @@ -206,10 +209,10 @@ public void testIngestTest() { } // run the ingest - doIngestTable(schemaOwner, testTable); + doIngestTable(schemaOwner, testTable, ExecutionPhase.COMPLETED); // cleanup on success - doDelete(schemaOwner, td.getTableName(), false); + doDelete(schemaOwner, testTable, false); } catch (Exception unexpected) { log.error("unexpected exception", unexpected); Assert.fail("unexpected exception: " + unexpected); @@ -243,7 +246,7 @@ public void testIngestUnsupportedDataType() { log.debug("created database table: " + testTable); try { - doIngestTable(schemaOwner, testTable); + doIngestTable(schemaOwner, testTable, ExecutionPhase.ERROR); } catch (UnsupportedOperationException expected) { log.info("expected exception: " + expected); } @@ -256,7 +259,7 @@ public void testIngestUnsupportedDataType() { } } - void doIngestTable(Subject subject, String tableName) throws Exception { + void doIngestTable(Subject subject, String tableName, ExecutionPhase expected) throws Exception { // create job Map params = new TreeMap(); params.put("op", "ingest"); @@ -296,8 +299,7 @@ void doIngestTable(Subject subject, String tableName) throws Exception { JobReader r = new JobReader(); Job end = r.read(new StringReader(xml2)); - log.debug(end.getErrorSummary().toString()); -// Assert.assertEquals("final job state", ExecutionPhase.COMPLETED, end.getExecutionPhase()); + Assert.assertEquals("final job state", expected, end.getExecutionPhase()); } } From 27f478219bd90492c7136f40944420b60ef4ff40 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 12 Sep 2024 13:08:59 -0700 Subject: [PATCH 12/14] CADC-13452 TableUpdateTest updates --- .../intTest/java/org/opencadc/youcat/TableUpdateTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java index 8f3f778f..a8b54327 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/TableUpdateTest.java @@ -194,11 +194,8 @@ public void testIngestTable() { TapPermissions tp = new TapPermissions(null, true, null, null); super.setPerms(schemaOwner, testSchemaName, tp, 200); - // cleanup - String testTable = testSchemaName + ".test_ingest_table"; - doDelete(schemaOwner, testTable, true); - // create test table and schema + String testTable = testSchemaName + ".test_ingest_table"; TableDesc td = doCreateTable(schemaOwner, testTable); // delete the schema from tap_schema @@ -262,7 +259,7 @@ public void testIngestUnsupportedDataType() { void doIngestTable(Subject subject, String tableName, ExecutionPhase expected) throws Exception { // create job Map params = new TreeMap(); - params.put("op", "ingest"); + params.put("ingest", "true"); params.put("table", tableName); HttpPost post = new HttpPost(certUpdateURL, params, false); Subject.doAs(subject, new RunnableAction(post)); From 7234013cfd570e9173905bd33961d021c7c9a5ba Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 12 Sep 2024 17:20:24 -0700 Subject: [PATCH 13/14] CADC-13452 int-test fixes --- .../ca/nrc/cadc/tap/db/TableIngesterTest.java | 15 +++---- .../ca/nrc/cadc/tap/pg/TableIngesterTest.java | 39 +++++++++---------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java index 86dc19bc..efc60856 100644 --- a/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java +++ b/cadc-tap-schema/src/intTest/java/ca/nrc/cadc/tap/db/TableIngesterTest.java @@ -323,6 +323,7 @@ public void testUnsupportedDataType() throws Exception { Assert.fail("unexpected exception: " + unexpected); } } + @Ignore @Test public void testPrintTableMetadata() { @@ -396,14 +397,14 @@ public void testPrintTableMetadata() { } } } - } - // cleanup - try { - tableCreator.dropTable(testTable); - log.debug("dropped table: " + testTable); - } catch (Exception ignore) { - log.debug("database-cleanup-after-test failed for " + testTable); + // cleanup + try { + tableCreator.dropTable(testTable); + log.debug("dropped table: " + testTable); + } catch (Exception ignore) { + log.debug("database-cleanup-after-test failed for " + testTable); + } } } catch (Exception unexpected) { log.error("unexpected exception", unexpected); diff --git a/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java index 5c4c54e4..5f7b85f7 100644 --- a/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java +++ b/cadc-tap-server-pg/src/intTest/java/ca/nrc/cadc/tap/pg/TableIngesterTest.java @@ -97,13 +97,13 @@ public class TableIngesterTest { private static final Logger log = Logger.getLogger(TableIngesterTest.class); static { - Log4jInit.setLevel("ca.nrc.cadc.tap.db", Level.DEBUG); - Log4jInit.setLevel("ca.nrc.cadc.tap.schema", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.tap.db", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap.schema", Level.INFO); } private final DataSource dataSource; private final TapSchemaDAO tapSchemaDAO; - private final String TEST_SCHEMA = "tap_schema"; + private final String TEST_SCHEMA = "int_test_schema"; public TableIngesterTest() { // create a datasource and register with JNDI @@ -271,13 +271,13 @@ public void testPrintTableMetadata() { try { tableCreator.dropTable(testTable); } catch (Exception ignore) { - log.debug("database-cleanup-before-test failed for " + testTable); + log.info("database-cleanup-before-test failed for " + testTable); } // create test table in the database TableDesc ingestTable = getTableDesc(TEST_SCHEMA, testTable); tableCreator.createTable(ingestTable); - log.debug("created database table: " + testTable); + log.info("created database table: " + testTable); List columnInfo = new ArrayList() {{ add("DATA_TYPE"); @@ -302,14 +302,11 @@ public void testPrintTableMetadata() { }}; try (Connection connection = dataSource.getConnection()) { - // create two indexes on the table + // create an index on the table JdbcTemplate jdbc = new JdbcTemplate(dataSource); - String index1 = "CREATE UNIQUE INDEX c0_idx ON int_test_schema.testPrintTableMetadata (c0)"; - log.debug("sql:\n" + index1); - jdbc.execute(index1); - String index2 = "CREATE UNIQUE INDEX c6_idx ON int_test_schema.testPrintTableMetadata (c6)"; - log.debug("sql:\n" + index2); - jdbc.execute(index2); + String index = "CREATE UNIQUE INDEX a11_idx ON int_test_schema.testPrintTableMetadata (a11)"; + log.info("sql:\n" + index); + jdbc.execute(index); log.info("column metadata"); DatabaseMetaData metaData = connection.getMetaData(); @@ -333,14 +330,14 @@ public void testPrintTableMetadata() { } } } - } - // cleanup - try { - tableCreator.dropTable(testTable); - log.debug("dropped table: " + testTable); - } catch (Exception ignore) { - log.debug("database-cleanup-after-test failed for " + testTable); + // cleanup + try { + tableCreator.dropTable(testTable); + log.info("dropped table: " + testTable); + } catch (Exception ignore) { + log.info("database-cleanup-after-test failed for " + testTable); + } } } catch (Exception unexpected) { log.error("unexpected exception", unexpected); @@ -355,8 +352,8 @@ TableDesc getTableDesc(String schemaName, String tableName) throws Exception { tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e9", TapDataType.CIRCLE)); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "e10", TapDataType.POLYGON)); // arrays - tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a11", new TapDataType("short", "*", null))); - tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a12", new TapDataType("int", "*", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a11", new TapDataType("short", "8", null))); + tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a12", new TapDataType("int", "*12", null))); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a13", new TapDataType("long", "*", null))); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a14", new TapDataType("float", "*", null))); tableDesc.getColumnDescs().add(new ColumnDesc(tableName, "a15", new TapDataType("double", "*", null))); From 5538de4521c484dd47fc7203a2a11f48f2605559 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 12 Sep 2024 17:21:28 -0700 Subject: [PATCH 14/14] CADC-13452 add api_created=true to tables created with TableIngester --- .../src/main/java/ca/nrc/cadc/tap/db/TableIngester.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java index 152f6eb1..cc75043c 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableIngester.java @@ -139,14 +139,10 @@ public void ingest(String schemaName, String tableName) { try { tm.startTransaction(); + // TODO: change getSchema() above to lockSchema() once implemented to prevent duplicate put // add the schema to the tap_schema if it doesn't exist SchemaDesc schemaDesc = tapSchemaDAO.getSchema(schemaName, true); - if (schemaDesc == null) { - schemaDesc = new SchemaDesc(schemaName); - schemaDesc.tapPermissions = tapPermissions; - tapSchemaDAO.put(schemaDesc); - log.debug(String.format("added schema '%s' to tap_schema", schemaDesc.getSchemaName())); - } else { + if (schemaDesc != null) { log.debug(String.format("existing schema '%s' in tap_schema", schemaDesc.getSchemaName())); } @@ -203,6 +199,7 @@ protected TableDesc createTableDesc(String schemaName, String tableName) // build TableDesc TableDesc tableDesc = new TableDesc(schemaName, tableName); tableDesc.tableType = TableDesc.TableType.TABLE; + tableDesc.apiCreated = true; log.debug(String.format("creating TableDesc %s %s", schemaName, tableName)); while (columnInfo.next()) { String columnName = columnInfo.getString("COLUMN_NAME");