diff --git a/csv_ingestor/src/csv_ingest.c b/csv_ingestor/src/csv_ingest.c index fad1d12..e273e2e 100644 --- a/csv_ingestor/src/csv_ingest.c +++ b/csv_ingestor/src/csv_ingest.c @@ -8,9 +8,9 @@ ******************************************************************************** * * REPOSITORY INFORMATION: -* $Revision: 66144 $ +* $Revision: 66395 $ * $Author: ermold $ -* $Date: 2015-12-01 18:56:24 +0000 (Tue, 01 Dec 2015) $ +* $Date: 2015-12-10 23:53:00 +0000 (Thu, 10 Dec 2015) $ * ******************************************************************************** * @@ -24,7 +24,7 @@ #include "csv_ingest.h" -static char *gVersion = "$State: ingest-csv_ingestor-1.0-beta $"; +static char *gVersion = "$State: ingest-csv_ingestor-1.0 $"; /** * Initialize the CSV process. @@ -44,76 +44,34 @@ static char *gVersion = "$State: ingest-csv_ingestor-1.0-beta $"; */ void *csv_ingest_init(void) { - UserData *data = (UserData *)NULL; - CSVConf *conf = (CSVConf *)NULL; - CSVParser *csv = (CSVParser *)NULL; - int ndsids; - int *dsids; - const char *level; - int status; - SplitMode split_mode; - double split_start; - double split_interval; + UserData *data = (UserData *)NULL; + CSVConf *conf = (CSVConf *)NULL; + int *dsids = (int *)NULL; + int ndsids; + DsData **dsp; + const char *name; + const char *level; + int dsi; /************************************************************ * Create the UserData structure *************************************************************/ DSPROC_DEBUG_LV1( - "Creating user defined data structure\n"); + "Initializing csv_ingestor process\n"); data = (UserData *)calloc(1, sizeof(UserData)); if (!data) { - DSPROC_ERROR( DSPROC_ENOMEM, - "Memory allocation error creating user data structure\n"); - - goto ERROR_EXIT; + "Memory allocation error initializing process\n"); + return((void *)data); } data->proc_name = dsproc_get_name(); data->site = dsproc_get_site(); data->fac = dsproc_get_facility(); - /************************************************************ - * Find and load the CSV Ingest configuration file - *************************************************************/ - - conf = dsproc_init_csv_conf(data->proc_name, NULL); - if (!conf) { - goto ERROR_EXIT; - } - - status = dsproc_load_csv_conf(conf, 0, CSV_CHECK_DATA_CONF); - if (status <= 0) { - - if (status == 0) { - - DSPROC_ERROR( NULL, - "Could not find required configuration file"); - } - - goto ERROR_EXIT; - } - - data->conf = conf; - - /************************************************************ - * Initialize the CSV Parser - *************************************************************/ - - csv = dsproc_init_csv_parser(NULL); - if (!csv) { - goto ERROR_EXIT; - } - - if (!dsproc_configure_csv_parser(conf, csv)) { - goto ERROR_EXIT; - } - - data->csv = csv; - /********************************************************************* * Get the input datastream ID. **********************************************************************/ @@ -123,71 +81,49 @@ void *csv_ingest_init(void) if (ndsids != 1) { if (ndsids > 1) { + DSPROC_ERROR( NULL, - "Too Many Input Datastreams Defined For Process: %s" - " -> found %d, but only 1 was expected", + "Too many input datastreams defined for process: %s" + " -> found %d, but only expected one", data->proc_name, ndsids); } else if (ndsids == 0) { + DSPROC_ERROR( NULL, - "No Input Datastreams Defined For Process: %s", + "No input datastreams defined for process: %s", data->proc_name); } - free(dsids); + if (dsids) free(dsids); goto ERROR_EXIT; } - data->in_dsid = dsids[0]; + data->raw_in_dsid = dsids[0]; free(dsids); - - /************************************************************ - * Add the file patterns to look for to the input datastream - *************************************************************/ - - if (!conf->fn_npatterns) { - - DSPROC_ERROR( NULL, - "No Input File Name Patterns Found In CSV Configuration File: %s/%s", - conf->file_path, conf->file_name); - - goto ERROR_EXIT; - } - - if (!dsproc_add_datastream_file_patterns( - data->in_dsid, conf->fn_npatterns, conf->fn_patterns, 0)) { - - goto ERROR_EXIT; - } + dsids = (int *)NULL; /*********************************************************** * Get the output datastream IDs **************************************************************/ ndsids = dsproc_get_output_datastream_ids(&dsids); - if (ndsids != 2) { + if (ndsids < 2) { - if (ndsids > 2) { - DSPROC_ERROR( NULL, - "Too Many Output Datastreams Defined For Process: %s", - " -> expected 2 but found %d", - data->proc_name, ndsids); - } - else if (ndsids == 1) { + if (ndsids == 1) { level = dsproc_datastream_class_level(dsids[0]); if (*level == '0') { DSPROC_ERROR( NULL, - "Not Enough Output Datastreams Defined For Process: %s" + "Not enough output datastreams defined for process: %s" " -> missing output datastream for processed data", data->proc_name); } else { DSPROC_ERROR( NULL, - "Not Enough Output Datastreams Defined For Process: %s" + "Not enough output datastreams defined for process: %s" " -> missing output datastream for raw data", data->proc_name); } @@ -195,112 +131,99 @@ void *csv_ingest_init(void) else if (ndsids == 0) { DSPROC_ERROR( NULL, - "No Output Datastreams Defined For Process: %s", + "No output datastreams defined for process: %s", data->proc_name); } - free(dsids); + if (dsids) free(dsids); goto ERROR_EXIT; } - level = dsproc_datastream_class_level(dsids[0]); - - if (*level == '0') { + /*********************************************************** + * Initialize the Datastream Data structures + **************************************************************/ - data->raw_dsid = dsids[0]; + dsp = calloc(ndsids, sizeof(DsData *)); + if (!dsp) { + DSPROC_ERROR( DSPROC_ENOMEM, + "Memory allocation error initializing process\n"); + goto ERROR_EXIT; + } - level = dsproc_datastream_class_level(dsids[1]); + data->dsp = dsp; + data->ndsp = 0; + data->raw_out_dsid = -1; - if (*level != '0') { - data->out_dsid = dsids[1]; - } - else { + for (dsi = 0; dsi < ndsids; ++dsi) { - DSPROC_ERROR( NULL, - "Invalid Output Datastream Defined For Process: %s" - " -> a process can only have one output datastream for raw data", - data->proc_name); + name = dsproc_datastream_class_name(dsids[dsi]); + level = dsproc_datastream_class_level(dsids[dsi]); - free(dsids); - free(data); - return((void *)NULL); - } - } - else { + if (*level == '0') { - data->out_dsid = dsids[0]; + if (data->raw_out_dsid == -1) { + data->raw_out_dsid = dsids[dsi]; + } + else { - level = dsproc_datastream_class_level(dsids[1]); + DSPROC_ERROR( NULL, + "Invalid output datastream: %s.%s" + " -> a process can only have one output 00 level datastream for raw data", + name, level); - if (*level == '0') { - data->raw_dsid = dsids[1]; + free(dsids); + goto ERROR_EXIT; + } } else { - - DSPROC_ERROR( NULL, - "Invalid Output Datastream Defined For Process: %s" - " -> a process must have one output datastream for raw data", - data->proc_name); - - free(dsids); - free(data); - return((void *)NULL); + dsp[data->ndsp] = csv_ingest_init_dsdata(name, level); + if (!dsp[data->ndsp]) goto ERROR_EXIT; + data->ndsp += 1; } } free(dsids); + dsids = (int *)NULL; - /************************************************************ - * Check if a split interval was set in the conf file - *************************************************************/ + if (data->raw_out_dsid == -1) { - if (conf->split_interval) { + DSPROC_ERROR( NULL, + "Not enough output datastreams defined for process: %s" + " -> a process must have one output 00 level datastream for raw data", + data->proc_name); - if (strcmp(conf->split_interval, "DAILY") == 0) { + goto ERROR_EXIT; + } - split_mode = SPLIT_ON_HOURS; - split_start = 0.0; - split_interval = 24.0; - } - else if (strcmp(conf->split_interval, "MONTHLY") == 0) { + /************************************************************ + * Add the input file patterns to look for + *************************************************************/ - split_mode = SPLIT_ON_MONTHS; - split_start = 1.0; - split_interval = 1.0; - } - else if (strcmp(conf->split_interval, "YEARLY") == 0) { + for (dsi = 0; dsi < data->ndsp; ++dsi) { - split_mode = SPLIT_ON_MONTHS; - split_start = 1.0; - split_interval = 12.0; - } - else if (strcmp(conf->split_interval, "FILE") == 0) { + conf = data->dsp[dsi]->conf; - split_mode = SPLIT_ON_STORE; - split_start = 0.0; - split_interval = 0.0; - } - else { + if (!conf->fn_npatterns) { DSPROC_ERROR( NULL, - "Invalid Split Interval Defined In: %s/%s", + "No input file name patterns found in configuration file: %s/%s", conf->file_path, conf->file_name); - free(data); - return((void *)NULL); + goto ERROR_EXIT; } - dsproc_set_datastream_split_mode( - data->out_dsid, split_mode, split_start, split_interval); + if (!dsproc_add_datastream_file_patterns( + data->raw_in_dsid, conf->fn_npatterns, conf->fn_patterns, 0)) { + + goto ERROR_EXIT; + } } return((void *)data); ERROR_EXIT: - if (data) free(data); - if (conf) dsproc_free_csv_conf(conf); - if (csv) dsproc_free_csv_parser(csv); + csv_ingest_finish(data); return((void *)NULL); } @@ -316,14 +239,21 @@ void *csv_ingest_init(void) void csv_ingest_finish(void *user_data) { UserData *data = (UserData *)user_data; + int dsi; DSPROC_DEBUG_LV1( - "Cleaning up user defined data structure\n"); + "Cleaning up allocated memory\n"); if (data) { - if (data->conf) dsproc_free_csv_conf(data->conf); - if (data->csv) dsproc_free_csv_parser(data->csv); + if (data->dsp) { + + for (dsi = 0; dsi < data->ndsp; ++dsi) { + csv_ingest_free_dsdata(data->dsp[dsi]); + } + + free(data->dsp); + } free(data); } @@ -349,14 +279,53 @@ int csv_ingest_process_file( const char *file_name) { UserData *data = (UserData *)user_data; - CSVConf *conf = data->conf; - CSVParser *csv = data->csv; + DsData *ds = (DsData *)NULL; + CSVConf *conf; + CSVParser *csv; time_t file_time; char full_path[PATH_MAX]; int nrecs_loaded; int nrecs_stored; int status; + int dsi; + + /************************************************************ + * Find the correct DsData structure to use for this file + *************************************************************/ + + for (dsi = 0; dsi < data->ndsp; ++dsi) { + + ds = data->dsp[dsi]; + + status = relist_execute( + ds->fn_relist, file_name, 0, NULL, NULL, NULL, NULL); + + if (status == 1) { + break; + } + + if (status < 0) { + + DSPROC_ERROR( NULL, + "Regex error while looking for DsData structure for file: %s\n", + file_name); + + return(-1); + } + } + + if (dsi == data->ndsp) { + + DSPROC_ERROR( NULL, + "Could not find matching DsData structure for file: %s\n", + file_name); + + return(-1); + } + + conf = ds->conf; + csv = ds->csv; /************************************************************ * Initialize data structures for a new file read @@ -372,7 +341,7 @@ int csv_ingest_process_file( /* Set the number of dots from the end of the file name to * preserve when the file is renamed. */ - if (!dsproc_set_preserve_dots_from_name(data->raw_dsid, file_name)) { + if (!dsproc_set_preserve_dots_from_name(data->raw_out_dsid, file_name)) { return(-1); } @@ -393,6 +362,13 @@ int csv_ingest_process_file( if (!dsproc_configure_csv_parser(conf, csv)) { return(-1); } + + /* Free the csv to cds map so it gets recreated. */ + + if (ds->map) { + dsproc_free_csv_to_cds_map(ds->map); + ds->map = (CSV2CDSMap *)NULL; + } } /************************************************************ @@ -403,7 +379,7 @@ int csv_ingest_process_file( snprintf(full_path, PATH_MAX, "%s/%s", input_dir, file_name); - nrecs_loaded = csv_ingest_read_data(data); + nrecs_loaded = csv_ingest_read_data(data, ds); if (nrecs_loaded < 0) return(-1); if (nrecs_loaded == 0) { @@ -415,7 +391,7 @@ int csv_ingest_process_file( data->begin_time = file_time; } - if (dsproc_rename_bad(data->raw_dsid, + if (dsproc_rename_bad(data->raw_out_dsid, input_dir, file_name, data->begin_time)) { return(0); @@ -429,14 +405,14 @@ int csv_ingest_process_file( * Store the data *************************************************************/ - nrecs_stored = csv_ingest_store_data(data); + nrecs_stored = csv_ingest_store_data(data, ds); if (nrecs_stored < 0) return(-1); /************************************************************ * Rename the raw data file *************************************************************/ - if (!dsproc_rename(data->raw_dsid, + if (!dsproc_rename(data->raw_out_dsid, input_dir, file_name, data->begin_time, data->end_time)) { return(-1); @@ -445,6 +421,194 @@ int csv_ingest_process_file( return(1); } +/** + * Free the memory used by a DsData structure. + * + * @param ds pointer to the DsData structure + */ +void csv_ingest_free_dsdata(DsData *ds) +{ + if (ds) { + + if (ds->conf) dsproc_free_csv_conf(ds->conf); + if (ds->csv) dsproc_free_csv_parser(ds->csv); + if (ds->map) dsproc_free_csv_to_cds_map(ds->map); + if (ds->fn_relist) relist_free(ds->fn_relist); + + free(ds); + } +} + +/** + * Initialize a Datastream Data structure. + * + * If an error occurs in this function it will be appended to the log and + * error mail messages, and the process status will be set appropriately. + * + * @param dsname output datastream name + * @param dslevel output datastream level + * + * @retval ds pointer to the new DsData structure + * @retval NULL if an error occurred + */ +DsData *csv_ingest_init_dsdata( + const char *dsname, + const char *dslevel) +{ + DsData *ds = (DsData *)NULL; + CSVConf *conf = (CSVConf *)NULL; + CSVParser *csv = (CSVParser *)NULL; + REList *fn_relist = (REList *)NULL; + int re_cflags = REG_EXTENDED | REG_NOSUB; + SplitMode split_mode; + double split_start; + double split_interval; + int status; + + /************************************************************ + * Allocate memory for the new DsData structure + *************************************************************/ + + ds = (DsData *)calloc(1, sizeof(DsData)); + if (!ds) { + DSPROC_ERROR( DSPROC_ENOMEM, + "Memory allocation error creating DsData structure\n"); + return(0); + } + + /*********************************************************** + * Get the output datastream ID + **************************************************************/ + + ds->dsid = dsproc_get_output_datastream_id(dsname, dslevel); + if (ds->dsid < 0) { + goto ERROR_EXIT; + } + + /************************************************************ + * Find and load the CSV Ingest configuration file + *************************************************************/ + + conf = dsproc_init_csv_conf(dsname, dslevel); + if (!conf) { + goto ERROR_EXIT; + } + + status = dsproc_load_csv_conf(conf, 0, CSV_CHECK_DATA_CONF); + if (status <= 0) { + + if (status == 0) { + + DSPROC_ERROR( NULL, + "Could not find required configuration file: %s.%s.csv_conf", + dsname, dslevel); + } + + goto ERROR_EXIT; + } + + ds->conf = conf; + + /************************************************************ + * Initialize the CSV Parser + *************************************************************/ + + csv = dsproc_init_csv_parser(NULL); + if (!csv) { + goto ERROR_EXIT; + } + + if (!dsproc_configure_csv_parser(conf, csv)) { + goto ERROR_EXIT; + } + + ds->csv = csv; + + /************************************************************ + * Check if a split interval was set in the conf file + *************************************************************/ + + if (conf->split_interval) { + + if (strcmp(conf->split_interval, "DAILY") == 0) { + + split_mode = SPLIT_ON_HOURS; + split_start = 0.0; + split_interval = 24.0; + } + else if (strcmp(conf->split_interval, "MONTHLY") == 0) { + + split_mode = SPLIT_ON_MONTHS; + split_start = 1.0; + split_interval = 1.0; + } + else if (strcmp(conf->split_interval, "YEARLY") == 0) { + + split_mode = SPLIT_ON_MONTHS; + split_start = 1.0; + split_interval = 12.0; + } + else if (strcmp(conf->split_interval, "FILE") == 0) { + + split_mode = SPLIT_ON_STORE; + split_start = 0.0; + split_interval = 0.0; + } + else if (strcmp(conf->split_interval, "NONE") == 0) { + + split_mode = SPLIT_NONE; + split_start = 0.0; + split_interval = 0.0; + } + else { + + DSPROC_ERROR( NULL, + "Invalid split interval '%s' found in configuration file: %s/%s", + conf->split_interval, conf->file_path, conf->file_name); + + goto ERROR_EXIT; + } + + dsproc_set_datastream_split_mode( + ds->dsid, split_mode, split_start, split_interval); + } + + /************************************************************ + * Compile the file name patterns. This will be used later + * to determine which DsData structure to use. + *************************************************************/ + + if (!conf->fn_npatterns) { + + DSPROC_ERROR( NULL, + "No input file name patterns found in configuration file: %s/%s", + conf->file_path, conf->file_name); + + goto ERROR_EXIT; + } + + fn_relist = relist_compile( + NULL, conf->fn_npatterns, conf->fn_patterns, re_cflags); + + if (!fn_relist) { + + DSPROC_ERROR( NULL, + "Could not compile file name patterns in configuration file: %s/%s\n", + conf->file_path, conf->file_name); + + goto ERROR_EXIT; + } + + ds->fn_relist = fn_relist; + + return(ds); + +ERROR_EXIT: + + csv_ingest_free_dsdata(ds); + return((DsData *)NULL); +} + /** * Main CSV Ingest entry function. * diff --git a/csv_ingestor/src/csv_ingest.h b/csv_ingestor/src/csv_ingest.h index b9ccaa7..a8e82aa 100644 --- a/csv_ingestor/src/csv_ingest.h +++ b/csv_ingestor/src/csv_ingest.h @@ -8,9 +8,9 @@ ******************************************************************************** * * REPOSITORY INFORMATION: -* $Revision: 65934 $ +* $Revision: 66394 $ * $Author: ermold $ -* $Date: 2015-11-18 19:24:30 +0000 (Wed, 18 Nov 2015) $ +* $Date: 2015-12-10 23:49:40 +0000 (Thu, 10 Dec 2015) $ * ******************************************************************************** * @@ -32,31 +32,45 @@ */ /*@{*/ +/** + * Datastream Data structure + * + * Structure used to read in an input file and map it to an output dataset. + */ +typedef struct +{ + int dsid; /**< output datastream ID */ + CSVConf *conf; /**< CSV configuration structure */ + CSVParser *csv; /**< pointer to the CSV Parser */ + CSV2CDSMap *map; /**< pointer to the CSV to CDS mapping structure */ + REList *fn_relist; /**< compiled file name patterns */ + +} DsData; + /** * UserData structure passed to all user defined hook functions. */ typedef struct { - const char *proc_name; /**< process name */ - const char *site; /**< process site */ - const char *fac; /**< process facility */ + const char *proc_name; /**< process name */ + const char *site; /**< process site */ + const char *fac; /**< process facility */ - CSVConf *conf; /**< CSV configuration structure */ - CSVParser *csv; /**< pointer to the CSV Parser */ + int raw_in_dsid; /**< raw data input datastream ID */ + int raw_out_dsid; /**< raw data output datastream ID */ - int in_dsid; /**< input datastream ID */ - int out_dsid; /**< output datastream ID */ - int raw_dsid; /**< raw output datastream ID */ + const char *input_dir; /**< raw file input directory */ + const char *file_name; /**< name of the file being processed */ - const char *input_dir; /**< raw file input directory */ - const char *file_name; /**< name of the file being processed */ + time_t begin_time; /**< first record time in the current file */ + time_t end_time; /**< last record time in the current file */ - time_t begin_time; /**< first record time in the current file */ - time_t end_time; /**< last record time in the current file */ + DsData **dsp; /**< array of DsData pointers */ + int ndsp; /**< number of DsData entries */ } UserData; -/* Main Ingest Functions */ +/* Main CSV Ingest Functions */ void *csv_ingest_init(void); void csv_ingest_finish(void *user_data); @@ -65,25 +79,31 @@ int csv_ingest_process_file( const char *input_dir, const char *file_name); +void csv_ingest_free_dsdata(DsData *ds); + +DsData *csv_ingest_init_dsdata( + const char *dsname, + const char *dslevel); + int main(int argc, char *argv[]); /*@}*/ /** - * @defgroup CSV2NC_READ_DATA CSV2NC Read Data + * @defgroup CSV_INGESTOR_READ_DATA CSV Ingest Read Data */ /*@{*/ -int csv_ingest_read_data(UserData *data); +int csv_ingest_read_data(UserData *data, DsData *ds); /*@}*/ /** - * @defgroup CSV2NC_STORE_DATA CSV2NC Store Data + * @defgroup CSV_INGESTOR_STORE_DATA CSV Ingest Store Data */ /*@{*/ -int csv_ingest_store_data(UserData *data); +int csv_ingest_store_data(UserData *data, DsData *ds); /*@}*/ diff --git a/csv_ingestor/src/csv_ingest_read_data.c b/csv_ingestor/src/csv_ingest_read_data.c index a2641b4..72b215d 100644 --- a/csv_ingestor/src/csv_ingest_read_data.c +++ b/csv_ingestor/src/csv_ingest_read_data.c @@ -8,9 +8,9 @@ * ******************************************************************************** * * REPOSITORY INFORMATION: -* $Revision: 65933 $ +* $Revision: 66394 $ * $Author: ermold $ -* $Date: 2015-11-18 19:18:57 +0000 (Wed, 18 Nov 2015) $ +* $Date: 2015-12-10 23:49:40 +0000 (Thu, 10 Dec 2015) $ * ******************************************************************************** * @@ -31,14 +31,15 @@ * error mail messages, and the process status will be set appropriately. * * @param data pointer to the UserData structure + * @param ds pointer to the DsData structure * * @retval nrecs number of records read in * @retval -1 if a fatal error occurred */ -int csv_ingest_read_data(UserData *data) +int csv_ingest_read_data(UserData *data, DsData *ds) { - CSVConf *conf = data->conf; - CSVParser *csv = data->csv; + CSVConf *conf = ds->conf; + CSVParser *csv = ds->csv; char delim = conf->delim; char exp_ncols = conf->exp_ncols; const char *delims = ",\t"; @@ -191,7 +192,7 @@ int csv_ingest_read_data(UserData *data) if (!delim) { - DSPROC_ERROR( "Could Not Determine Delimiter From Header Line", + DSPROC_ERROR( "Could not determine delimiter from header line", "Could not determine delimiter from header line: '%s'\n", linep); diff --git a/csv_ingestor/src/csv_ingest_store_data.c b/csv_ingestor/src/csv_ingest_store_data.c index f9da410..85af485 100644 --- a/csv_ingestor/src/csv_ingest_store_data.c +++ b/csv_ingestor/src/csv_ingest_store_data.c @@ -8,9 +8,9 @@ * ******************************************************************************** * * REPOSITORY INFORMATION: -* $Revision: 65933 $ +* $Revision: 66394 $ * $Author: ermold $ -* $Date: 2015-11-18 19:18:57 +0000 (Wed, 18 Nov 2015) $ +* $Date: 2015-12-10 23:49:40 +0000 (Thu, 10 Dec 2015) $ * ******************************************************************************** * @@ -24,28 +24,6 @@ #include "csv_ingest.h" -/** - * Store metadata in the output dataset. - * - * If an error occurs in this function it will be appended to the log and - * error mail messages, and the process status will be set appropriately. - * - * @param data pointer to the UserData structure - * - * @retval 1 if successful - * @retval 0 if a fatal error occurred - */ -int csv_ingest_set_metadata(UserData *data) -{ - // Prevent unused parameter warnings, these - // can be removed if this function is ever used - data = data; - - // Add code to set any extra metadata here - - return(1); -} - /** * Store data. * @@ -57,18 +35,21 @@ int csv_ingest_set_metadata(UserData *data) * @retval nrecs number of records stored * @retval -1 if a fatal error occurred */ -int csv_ingest_store_data(UserData *data) +int csv_ingest_store_data(UserData *data, DsData *ds) { - int dsid = data->out_dsid; - CSVConf *conf = data->conf; - CSVParser *csv = data->csv; + int dsid = ds->dsid; + CSVConf *conf = ds->conf; + CSVParser *csv = ds->csv; timeval_t *times = csv->tvs; int nrecs = csv->nrecs; CDSGroup *dataset; - CSV2CDSMap *map; int nstored; + // prevent "unused parameter" compiler warning + + data = data; + /************************************************************ * Create the output dataset *************************************************************/ @@ -76,30 +57,26 @@ int csv_ingest_store_data(UserData *data) dataset = dsproc_create_output_dataset(dsid, times[0].tv_sec, 1); if (!dataset) return(-1); - /* Set the times in the output dataset */ + /************************************************************ + * Map the CSV fields to the output dataset variables + *************************************************************/ - if (!dsproc_set_sample_timevals(dataset, 0, nrecs, times)) { - return(-1); + if (!ds->map) { + ds->map = dsproc_create_csv_to_cds_map(conf, csv, dataset, 0); } - /* Set the metadata in the output dataset */ - - if (!csv_ingest_set_metadata(data)) { + if (!dsproc_map_csv_to_cds(csv, 0, 0, ds->map, dataset, 0, 0)) { return(-1); } /************************************************************ - * Map the CSV fields to the output dataset variables + * Set times in output dataset *************************************************************/ - map = dsproc_create_csv_to_cds_map(conf, csv, dataset, 0); - - if (!dsproc_map_csv_to_cds(csv, 0, 0, map, dataset, 0, 0)) { + if (!dsproc_set_sample_timevals(dataset, 0, nrecs, times)) { return(-1); } - dsproc_free_csv_to_cds_map(map); - /************************************************************ * Store the output dataset *************************************************************/