Skip to content

Commit

Permalink
Updated redcap-etl to 2.0.11
Browse files Browse the repository at this point in the history
  • Loading branch information
@mullen2
mullen2 committed Nov 8, 2021
1 parent f3c48e2 commit 6b6f894
Show file tree
Hide file tree
Showing 14 changed files with 159 additions and 135 deletions.
2 changes: 1 addition & 1 deletion composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
],
"require" : {
"phpseclib/phpseclib" : "3.0.10",
"iu-redcap/redcap-etl" : "2.0.10"
"iu-redcap/redcap-etl" : "2.0.11"
},
"require-dev": {
"phpunit/phpunit": "9.5.8",
Expand Down
138 changes: 70 additions & 68 deletions composer.lock
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion dependencies/autoload.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInit8f5fd9a9eae22f7e6eea4e9a7724d62e::getLoader();
return ComposerAutoloaderInit0890d3004c704e958588b2b63dfa3f67::getLoader();
14 changes: 7 additions & 7 deletions dependencies/composer/autoload_real.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

// autoload_real.php @generated by Composer

class ComposerAutoloaderInit8f5fd9a9eae22f7e6eea4e9a7724d62e
class ComposerAutoloaderInit0890d3004c704e958588b2b63dfa3f67
{
private static $loader;

Expand All @@ -22,15 +22,15 @@ public static function getLoader()
return self::$loader;
}

spl_autoload_register(array('ComposerAutoloaderInit8f5fd9a9eae22f7e6eea4e9a7724d62e', 'loadClassLoader'), true, true);
spl_autoload_register(array('ComposerAutoloaderInit0890d3004c704e958588b2b63dfa3f67', 'loadClassLoader'), true, true);
self::$loader = $loader = new \Composer\Autoload\ClassLoader();
spl_autoload_unregister(array('ComposerAutoloaderInit8f5fd9a9eae22f7e6eea4e9a7724d62e', 'loadClassLoader'));
spl_autoload_unregister(array('ComposerAutoloaderInit0890d3004c704e958588b2b63dfa3f67', 'loadClassLoader'));

$useStaticLoader = PHP_VERSION_ID >= 50600 && !defined('HHVM_VERSION') && (!function_exists('zend_loader_file_encoded') || !zend_loader_file_encoded());
if ($useStaticLoader) {
require_once __DIR__ . '/autoload_static.php';

call_user_func(\Composer\Autoload\ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e::getInitializer($loader));
call_user_func(\Composer\Autoload\ComposerStaticInit0890d3004c704e958588b2b63dfa3f67::getInitializer($loader));
} else {
$map = require __DIR__ . '/autoload_namespaces.php';
foreach ($map as $namespace => $path) {
Expand All @@ -51,19 +51,19 @@ public static function getLoader()
$loader->register(true);

if ($useStaticLoader) {
$includeFiles = Composer\Autoload\ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e::$files;
$includeFiles = Composer\Autoload\ComposerStaticInit0890d3004c704e958588b2b63dfa3f67::$files;
} else {
$includeFiles = require __DIR__ . '/autoload_files.php';
}
foreach ($includeFiles as $fileIdentifier => $file) {
composerRequire8f5fd9a9eae22f7e6eea4e9a7724d62e($fileIdentifier, $file);
composerRequire0890d3004c704e958588b2b63dfa3f67($fileIdentifier, $file);
}

return $loader;
}
}

function composerRequire8f5fd9a9eae22f7e6eea4e9a7724d62e($fileIdentifier, $file)
function composerRequire0890d3004c704e958588b2b63dfa3f67($fileIdentifier, $file)
{
if (empty($GLOBALS['__composer_autoload_files'][$fileIdentifier])) {
require $file;
Expand Down
6 changes: 3 additions & 3 deletions dependencies/composer/autoload_static.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

namespace Composer\Autoload;

class ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e
class ComposerStaticInit0890d3004c704e958588b2b63dfa3f67
{
public static $files = array (
'decc78cc4436b1292c6c0d151b19445c' => __DIR__ . '/..' . '/phpseclib/phpseclib/phpseclib/bootstrap.php',
Expand Down Expand Up @@ -54,8 +54,8 @@ class ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e
public static function getInitializer(ClassLoader $loader)
{
return \Closure::bind(function () use ($loader) {
$loader->prefixLengthsPsr4 = ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInit8f5fd9a9eae22f7e6eea4e9a7724d62e::$prefixDirsPsr4;
$loader->prefixLengthsPsr4 = ComposerStaticInit0890d3004c704e958588b2b63dfa3f67::$prefixLengthsPsr4;
$loader->prefixDirsPsr4 = ComposerStaticInit0890d3004c704e958588b2b63dfa3f67::$prefixDirsPsr4;

}, null, ClassLoader::class);
}
Expand Down
14 changes: 7 additions & 7 deletions dependencies/composer/installed.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,17 @@
},
{
"name": "iu-redcap/redcap-etl",
"version": "2.0.10",
"version_normalized": "2.0.10.0",
"version": "2.0.11",
"version_normalized": "2.0.11.0",
"source": {
"type": "git",
"url": "https://github.com/IUREDCap/redcap-etl.git",
"reference": "0cddb601216fabc163e6e67749a73770e68dad11"
"reference": "0d03dabaaf3a84e7b7ced8afcb07c12895b69c00"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/IUREDCap/redcap-etl/zipball/0cddb601216fabc163e6e67749a73770e68dad11",
"reference": "0cddb601216fabc163e6e67749a73770e68dad11",
"url": "https://api.github.com/repos/IUREDCap/redcap-etl/zipball/0d03dabaaf3a84e7b7ced8afcb07c12895b69c00",
"reference": "0d03dabaaf3a84e7b7ced8afcb07c12895b69c00",
"shasum": ""
},
"require": {
Expand All @@ -69,9 +69,9 @@
"phpunit/phpunit": "9.5.8",
"squizlabs/php_codesniffer": "3.6.0"
},
"time": "2021-08-27T13:16:06+00:00",
"time": "2021-11-08T17:46:40+00:00",
"type": "library",
"installation-source": "source",
"installation-source": "dist",
"autoload": {
"psr-4": {
"IU\\REDCapETL\\": "src/"
Expand Down
2 changes: 1 addition & 1 deletion dependencies/iu-redcap/redcap-etl/src/Database/CsvDbConnection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ public function replaceLookupView($table, $lookupTable)
* Inserts the rows from the specified table
* into the database.
*/
protected function insertRows($table)
protected function insertRows($table, $batchSize = null)
{
$rows = $table->getRows();
foreach ($rows as $row) {
Expand Down
6 changes: 3 additions & 3 deletions dependencies/iu-redcap/redcap-etl/src/Database/DbConnection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ abstract public function insertRow($row);
* a description of the table where the rows are to be inserted,
* and the rows of data to be inserted.
*/
abstract protected function insertRows($table);
abstract protected function insertRows($table, $batchSize = null);

/**
* Process the specified file of queries.
Expand Down Expand Up @@ -172,9 +172,9 @@ public function storeRow($row)
return $rc;
}

public function storeRows($table)
public function storeRows($table, $batchSize = null)
{
$rc = $this->insertRows($table);
$rc = $this->insertRows($table, $batchSize);
return $rc;
}

Expand Down
54 changes: 32 additions & 22 deletions dependencies/iu-redcap/redcap-etl/src/Database/MysqlDbConnection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ public function insertRow($row)
*
* @param Table $table the table object containing the rows to be inserted in the database.
*/
protected function insertRows($table)
protected function insertRows($table, $batchSize = null)
{
$rows = $table->getRows();

Expand All @@ -446,31 +446,41 @@ protected function insertRows($table)
#--------------------------------------------------
$fields = $table->getAllNonAutoIncrementFields();

$queryValues = array();
foreach ($rows as $row) {
$rowValues = $this->getRowValues($row, $fields);
$queryValues[] = '('.implode(",", $rowValues).')';
$rowBatches = array();
if (isset($batchSize) && is_int($batchSize) && $batchSize > 0) {
$rowBatches = array_chunk($rows, $batchSize, true);
} else {
$rowBatches[] = $rows;
}

foreach ($rowBatches as $rows) {
$queryValues = array();

foreach ($rows as $row) {
$rowValues = $this->getRowValues($row, $fields);
$queryValues[] = '('.implode(",", $rowValues).')';
}

$query = $this->createInsertStatement($table->getName(), $fields, $queryValues);
# print "QUERY: {$query}\n";
$query = $this->createInsertStatement($table->getName(), $fields, $queryValues);
# print "QUERY: {$query}\n";

$rc = $this->mysqli->query($query);
$rc = $this->mysqli->query($query);

#---------------------------------------------------
# If there's an error executing the statement
#---------------------------------------------------
if ($rc === false) {
$this->errorString = $this->mysqli->error;
$result = false;

# Note: do not print out the specific query here, because it will
# be logged, and could contain PHI
$message = 'MySQL error while trying to insert values into table "'
.$this->escapeName($table->getName()).'": '
.' ['.$this->mysqli->errno.']: '.$this->mysqli->error;
$code = EtlException::DATABASE_ERROR;
throw new EtlException($message, $code);
#---------------------------------------------------
# If there's an error executing the statement
#---------------------------------------------------
if ($rc === false) {
$this->errorString = $this->mysqli->error;
$result = false;

# Note: do not print out the specific query here, because it will
# be logged, and could contain PHI
$message = 'MySQL error while trying to insert values into table "'
.$this->escapeName($table->getName()).'": '
.' ['.$this->mysqli->errno.']: '.$this->mysqli->error;
$code = EtlException::DATABASE_ERROR;
throw new EtlException($message, $code);
}
}
}

Expand Down
38 changes: 24 additions & 14 deletions dependencies/iu-redcap/redcap-etl/src/Database/PdoDbConnection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ public function insertRow($row)
*
* @param Table $table the table object containing the rows to be inserted in the database.
*/
protected function insertRows($table)
protected function insertRows($table, $batchSize = null)
{
$rows = $table->getRows();

Expand All @@ -342,22 +342,32 @@ protected function insertRows($table)
#--------------------------------------------------
$fields = $table->getAllNonAutoIncrementFields();

$queryValues = array();
foreach ($rows as $row) {
$rowValues = $this->getRowValues($row, $fields);
$queryValues[] = '('.implode(",", $rowValues).')';
$rowBatches = array();
if (isset($batchSize) && is_int($batchSize) && $batchSize > 0) {
$rowBatches = array_chunk($rows, $batchSize, true);
} else {
$rowBatches[] = $rows;
}

foreach ($rowBatches as $rows) {
$queryValues = array();

foreach ($rows as $row) {
$rowValues = $this->getRowValues($row, $fields);
$queryValues[] = '('.implode(",", $rowValues).')';
}

$query = $this->createInsertStatement($table->getName(), $fields, $queryValues);
#print "\n\nQUERY:\n----------------------------\n$query\n\n";
$query = $this->createInsertStatement($table->getName(), $fields, $queryValues);
#print "\n\nQUERY:\n----------------------------\n$query\n\n";

try {
$rc = $this->db->exec($query);
} catch (\Exception $exception) {
$message = 'Database error while trying to insert values into table "'
.$this->escapeName($table->getName()).'": '.$exception->getMessage();
$code = EtlException::DATABASE_ERROR;
throw new EtlException($message, $code);
try {
$rc = $this->db->exec($query);
} catch (\Exception $exception) {
$message = 'Database error while trying to insert values into table "'
.$this->escapeName($table->getName()).'": '.$exception->getMessage();
$code = EtlException::DATABASE_ERROR;
throw new EtlException($message, $code);
}
}
}

Expand Down
5 changes: 2 additions & 3 deletions dependencies/iu-redcap/redcap-etl/src/RulesGenerator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,9 +286,8 @@ protected function getFieldRule($field)
# so use a varchar here unless suitable min and max values are set
$min = $field['text_validation_min'];
$max = $field['text_validation_max'];
if (!empty($min) && !empty($max)
&& is_numeric($min) && is_numeric($max)
&& $min >= -2147483648 && $max <= 2147483647) { // min >= -(2^32); max <= 2^32 - 1
if (is_numeric($min) && is_numeric($max)
&& $min >= -2147483648 && $max <= 2147483647) { // min >= -(2^32); max <= 2^32 - 1
# MySQL int: [-2147483648, 2147483647] https://dev.mysql.com/doc/refman/5.7/en/integer-types.html
# PostgreSQL int: [-2147483648, 2147483647] https://www.postgresql.org/docs/9.1/datatype-numeric.html
# SQLite int: [-2147483648, 2147483647] (at least?) https://www.sqlite.org/datatype3.html
Expand Down
2 changes: 1 addition & 1 deletion dependencies/iu-redcap/redcap-etl/src/Version.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ class Version
# This release number should be updated each time a release is made.
# This value will be logged to indicate which version of the
# software is being used.
const RELEASE_NUMBER = "2.0.10";
const RELEASE_NUMBER = "2.0.11";
}
5 changes: 4 additions & 1 deletion dependencies/iu-redcap/redcap-etl/src/Workflow.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ class Workflow

private $workflowConfig;

/* Max batch size for metadata row insert; over 1,000 causes an error for SQL Server */
const METADATA_BATCH_SIZE = 1000;

/**
* Constructor.
*
Expand Down Expand Up @@ -374,7 +377,7 @@ public function createLoadTables($dbId, $schema)
$dbConnection->createTable($metadataTable, $ifNotExists);

$dbConnection->storeRows($projectInfoTable);
$dbConnection->storeRows($metadataTable);
$dbConnection->storeRows($metadataTable, self::METADATA_BATCH_SIZE);

#------------------------------------------------------------------------------------
# If configured, create the lookup table that maps multiple choice values to labels,
Expand Down
6 changes: 3 additions & 3 deletions dependencies/iu-redcap/redcap-etl/tests/data/projects/repeating-events-rules.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ FIELD,height_m,float
TABLE,cardiovascular,root,EVENTS
FIELD,cardiovascular_date,date
FIELD,hdl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,int
FIELD,triglycerides_mg_dl,varchar(255)
FIELD,diastolic1,varchar(255)
FIELD,diastolic2,varchar(255)
Expand All @@ -57,7 +57,7 @@ FIELD,systolic3,varchar(255)
TABLE,cardiovascular_repeating_events,root,REPEATING_EVENTS
FIELD,cardiovascular_date,date
FIELD,hdl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,int
FIELD,triglycerides_mg_dl,varchar(255)
FIELD,diastolic1,varchar(255)
FIELD,diastolic2,varchar(255)
Expand All @@ -69,7 +69,7 @@ FIELD,systolic3,varchar(255)
TABLE,cardiovascular_repeating_instruments,root,REPEATING_INSTRUMENTS
FIELD,cardiovascular_date,date
FIELD,hdl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,varchar(255)
FIELD,ldl_mg_dl,int
FIELD,triglycerides_mg_dl,varchar(255)
FIELD,diastolic1,varchar(255)
FIELD,diastolic2,varchar(255)
Expand Down

0 comments on commit 6b6f894

Please sign in to comment.