From 72ce217f9cf0917b8a71c391f41404e378f7bb39 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 24 Oct 2020 10:13:54 -0400 Subject: [PATCH] Optionally ES schema from file #117 --- docs/USAGE.md | 7 ++++++- src/cli.c | 38 ++++++++++++++++++++++++++++++-------- src/cli.h | 4 ++++ src/index/elastic.c | 10 +++++----- src/index/elastic.h | 2 +- src/main.c | 4 +++- 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/docs/USAGE.md b/docs/USAGE.md index 8ddc983..0e50c58 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -53,6 +53,8 @@ Index options --es-index= Elasticsearch index name. DEFAULT=sist2 -p, --print Just print JSON documents to stdout. --script-file= Path to user script. + --mappings-file= Path to Elasticsearch mappings. + --settings-file= Path to Elasticsearch settings. --async-script Execute user script asynchronously. --batch-size= Index batch size. DEFAULT: 100 -f, --force-reset Reset Elasticsearch mappings and settings. (You must use this option the first time you use the index command) @@ -257,6 +259,10 @@ it is currently unsupported and has no guaranties of back/forward compatibility. Print index in JSON format to stdout. * `--script-file` Path to user script. See [Scripting](scripting.md). + * `--mappings-file` + Path to custom Elasticsearch mappings. If none is specified, [the bundled mappings](https://github.com/simon987/sist2/tree/master/schema) will be used. + * `--settings-file` + Path to custom Elasticsearch settings. *(See above)* * `--async-script` Use `wait_for_completion=false` elasticsearch option while executing user script. (See [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html)) @@ -266,7 +272,6 @@ it is currently unsupported and has no guaranties of back/forward compatibility. down the process. * `-f, --force-reset` Reset Elasticsearch mappings and settings. - **(You must use this option the first time you use the index command)**. ### Index examples diff --git a/src/cli.c b/src/cli.c index 344c418..1cd8f2c 100644 --- a/src/cli.c +++ b/src/cli.c @@ -56,6 +56,12 @@ void scan_args_destroy(scan_args_t *args) { void index_args_destroy(index_args_t *args) { //todo + if (args->es_mappings_path) { + free(args->es_mappings); + } + if (args->es_settings_path) { + free(args->es_settings); + } free(args); } @@ -231,25 +237,25 @@ int scan_args_validate(scan_args_t *args, int argc, const char **argv) { return 0; } -int load_script(const char *script_path, char **dst) { +int load_external_file(const char *file_path, char **dst) { struct stat info; - int res = stat(script_path, &info); + int res = stat(file_path, &info); if (res == -1) { - fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno)); + LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) return 1; } - int fd = open(script_path, O_RDONLY); + int fd = open(file_path, O_RDONLY); if (fd == -1) { - fprintf(stderr, "Error opening script file '%s': %s\n", script_path, strerror(errno)); + LOG_ERRORF("cli.c", "Error opening file '%s': %s\n", file_path, strerror(errno)) return 1; } *dst = malloc(info.st_size + 1); res = read(fd, *dst, info.st_size); if (res < 0) { - fprintf(stderr, "Error reading script file '%s': %s\n", script_path, strerror(errno)); + LOG_ERRORF("cli.c", "Error reading file '%s': %s\n", file_path, strerror(errno)) return 1; } @@ -293,7 +299,19 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { } if (args->script_path != NULL) { - if (load_script(args->script_path, &args->script) != 0) { + if (load_external_file(args->script_path, &args->script) != 0) { + return 1; + } + } + + if (args->es_settings_path != NULL) { + if (load_external_file(args->es_settings_path, &args->es_settings) != 0) { + return 1; + } + } + + if (args->es_mappings_path != NULL) { + if (load_external_file(args->es_mappings_path, &args->es_mappings) != 0) { return 1; } } @@ -309,6 +327,10 @@ int index_args_validate(index_args_t *args, int argc, const char **argv) { LOG_DEBUGF("cli.c", "arg async_script=%s", args->async_script) LOG_DEBUGF("cli.c", "arg script=%s", args->script) LOG_DEBUGF("cli.c", "arg print=%d", args->print) + LOG_DEBUGF("cli.c", "arg es_mappings_path=%s", args->es_mappings_path) + LOG_DEBUGF("cli.c", "arg es_mappings=%s", args->es_mappings) + LOG_DEBUGF("cli.c", "arg es_settings_path=%s", args->es_settings_path) + LOG_DEBUGF("cli.c", "arg es_settings=%s", args->es_settings) LOG_DEBUGF("cli.c", "arg batch_size=%d", args->batch_size) LOG_DEBUGF("cli.c", "arg force_reset=%d", args->force_reset) @@ -445,7 +467,7 @@ int exec_args_validate(exec_args_t *args, int argc, const char **argv) { LOG_FATAL("cli.c", "--script-file argument is required"); } - if (load_script(args->script_path, &args->script) != 0) { + if (load_external_file(args->script_path, &args->script) != 0) { return 1; } diff --git a/src/cli.h b/src/cli.h index 71b6274..27139ef 100644 --- a/src/cli.h +++ b/src/cli.h @@ -39,6 +39,10 @@ typedef struct index_args { const char *index_path; const char *script_path; char *script; + const char *es_settings_path; + char *es_settings; + const char *es_mappings_path; + char *es_mappings; int print; int batch_size; int async_script; diff --git a/src/index/elastic.c b/src/index/elastic.c index 4645957..7d2b0a5 100644 --- a/src/index/elastic.c +++ b/src/index/elastic.c @@ -356,7 +356,7 @@ void finish_indexer(char *script, int async_script, char *index_id) { free_response(r); } -void elastic_init(int force_reset) { +void elastic_init(int force_reset, const char* user_mappings, const char* user_settings) { // Check if index exists char url[4096]; @@ -392,13 +392,13 @@ void elastic_init(int force_reset) { free_response(r); snprintf(url, sizeof(url), "%s/%s/_settings", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, settings_json); - LOG_INFOF("elastic.c", "Update settings <%d>", r->status_code); + r = web_put(url, user_settings ? user_settings : settings_json); + LOG_INFOF("elastic.c", "Update user_settings <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/%s/_mappings/_doc?include_type_name=true", IndexCtx.es_url, IndexCtx.es_index); - r = web_put(url, mappings_json); - LOG_INFOF("elastic.c", "Update mappings <%d>", r->status_code); + r = web_put(url, user_mappings ? user_mappings : mappings_json); + LOG_INFOF("elastic.c", "Update user_mappings <%d>", r->status_code); free_response(r); snprintf(url, sizeof(url), "%s/%s/_open", IndexCtx.es_url, IndexCtx.es_index); diff --git a/src/index/elastic.h b/src/index/elastic.h index ab1d7ac..c8c08a3 100644 --- a/src/index/elastic.h +++ b/src/index/elastic.h @@ -25,7 +25,7 @@ es_indexer_t *create_indexer(const char *url, const char *index); void elastic_cleanup(); void finish_indexer(char *script, int async_script, char *index_id); -void elastic_init(int force_reset); +void elastic_init(int force_reset, const char* user_mappings, const char* user_settings); cJSON *elastic_get_document(const char *uuid_str); diff --git a/src/main.c b/src/main.c index 957957c..199cf5f 100644 --- a/src/main.c +++ b/src/main.c @@ -267,7 +267,7 @@ void sist2_index(index_args_t *args) { IndexCtx.batch_size = args->batch_size; if (!args->print) { - elastic_init(args->force_reset); + elastic_init(args->force_reset, args->es_mappings, args->es_settings); } char descriptor_path[PATH_MAX]; @@ -446,6 +446,8 @@ int main(int argc, const char *argv[]) { OPT_STRING(0, "es-index", &common_es_index, "Elasticsearch index name. DEFAULT=sist2"), OPT_BOOLEAN('p', "print", &index_args->print, "Just print JSON documents to stdout."), OPT_STRING(0, "script-file", &common_script_path, "Path to user script."), + OPT_STRING(0, "mappings-file", &index_args->es_mappings_path, "Path to Elasticsearch mappings."), + OPT_STRING(0, "settings-file", &index_args->es_settings_path, "Path to Elasticsearch settings."), OPT_BOOLEAN(0, "async-script", &common_async_script, "Execute user script asynchronously."), OPT_INTEGER(0, "batch-size", &index_args->batch_size, "Index batch size. DEFAULT: 100"), OPT_BOOLEAN('f', "force-reset", &index_args->force_reset, "Reset Elasticsearch mappings and settings. "