From 75d2b6d99bd5f71322f1b78c94f5c292de174e91 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 15:26:48 -0800 Subject: [PATCH 01/20] Track an error state on yajl_buf --- ext/yajl/yajl_buf.c | 53 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index 04e608a3..d22bb719 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -38,35 +38,82 @@ #define YAJL_BUF_INIT_SIZE 2048 +typedef enum { + yajl_buf_ok = 0, + yajl_buf_alloc_failed, + yajl_buf_overflow +} yajl_buf_state; + struct yajl_buf_t { + yajl_buf_state state; unsigned int len; unsigned int used; unsigned char * data; yajl_alloc_funcs * alloc; }; +#include + +static +yajl_buf_state yajl_buf_set_error(yajl_buf buf, yajl_buf_state err) +{ + buf->state = err; + + // free and clear all data from the buffer + YA_FREE(buf->alloc, buf->data); + buf->len = 0; + buf->data = 0; + buf->used = 0; + + return err; +} + static -void yajl_buf_ensure_available(yajl_buf buf, unsigned int want) +yajl_buf_state yajl_buf_ensure_available(yajl_buf buf, unsigned int want) { unsigned int need; assert(buf != NULL); + if (buf->state != yajl_buf_ok) { + return buf->state; + } + /* first call */ if (buf->data == NULL) { buf->len = YAJL_BUF_INIT_SIZE; buf->data = (unsigned char *) YA_MALLOC(buf->alloc, buf->len); + if (buf->data == NULL) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + buf->data[0] = 0; } + if (want == 0) { + return yajl_buf_ok; + } + need = buf->len; while (want >= (need - buf->used)) need <<= 1; + // overflow + if (need == 0) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + if (need != buf->len) { buf->data = (unsigned char *) YA_REALLOC(buf->alloc, buf->data, need); + + if (buf->data == NULL) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + buf->len = need; } + + return yajl_buf_ok; } yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc) @@ -86,7 +133,9 @@ void yajl_buf_free(yajl_buf buf) void yajl_buf_append(yajl_buf buf, const void * data, unsigned int len) { - yajl_buf_ensure_available(buf, len); + if (yajl_buf_ensure_available(buf, len)) { + return; + } if (len > 0) { assert(data != NULL); memcpy(buf->data + buf->used, data, len); From 91e83845756a3e1daf020bda51773c66d7ab122a Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 15:40:32 -0800 Subject: [PATCH 02/20] Return an error from initial alloc failing --- ext/yajl/yajl_buf.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index d22bb719..b9177392 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -52,6 +52,11 @@ struct yajl_buf_t { yajl_alloc_funcs * alloc; }; +// A buffer to be returned if the initial allocation fails +static struct yajl_buf_t buf_alloc_error = { + .state = yajl_buf_alloc_failed +}; + #include static @@ -119,6 +124,10 @@ yajl_buf_state yajl_buf_ensure_available(yajl_buf buf, unsigned int want) yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc) { yajl_buf b = YA_MALLOC(alloc, sizeof(struct yajl_buf_t)); + if (b == NULL) { + return &buf_alloc_error; + } + memset((void *) b, 0, sizeof(struct yajl_buf_t)); b->alloc = alloc; return b; From a4a371f930f50639cbcfbb9e612c5aadee8be99d Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 15:49:51 -0800 Subject: [PATCH 03/20] Add assertions --- ext/yajl/yajl_buf.c | 17 +++++++++++------ ext/yajl/yajl_buf.h | 9 +++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index b9177392..cc8120af 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -38,12 +38,6 @@ #define YAJL_BUF_INIT_SIZE 2048 -typedef enum { - yajl_buf_ok = 0, - yajl_buf_alloc_failed, - yajl_buf_overflow -} yajl_buf_state; - struct yajl_buf_t { yajl_buf_state state; unsigned int len; @@ -59,6 +53,12 @@ static struct yajl_buf_t buf_alloc_error = { #include +yajl_buf_state yajl_buf_err(yajl_buf buf) +{ + assert(buf); + return buf->state; +} + static yajl_buf_state yajl_buf_set_error(yajl_buf buf, yajl_buf_state err) { @@ -161,17 +161,22 @@ void yajl_buf_clear(yajl_buf buf) const unsigned char * yajl_buf_data(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); return buf->data; } unsigned int yajl_buf_len(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); return buf->used; } void yajl_buf_truncate(yajl_buf buf, unsigned int len) { + assert(buf); assert(len <= buf->used); buf->used = len; } diff --git a/ext/yajl/yajl_buf.h b/ext/yajl/yajl_buf.h index f97355b9..f78f212f 100644 --- a/ext/yajl/yajl_buf.h +++ b/ext/yajl/yajl_buf.h @@ -43,6 +43,12 @@ * call overhead. YMMV. */ +typedef enum { + yajl_buf_ok = 0, + yajl_buf_alloc_failed, + yajl_buf_overflow +} yajl_buf_state; + /** * yajl_buf is a buffer with exponential growth. the buffer ensures that * you are always null padded. @@ -77,4 +83,7 @@ unsigned int yajl_buf_len(yajl_buf buf); YAJL_API void yajl_buf_truncate(yajl_buf buf, unsigned int len); +/* get the state of buffer */ +yajl_buf_state yajl_buf_err(yajl_buf buf); + #endif From e001a1c2ea8542ade790244cba3306bc56a54587 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 15:51:01 -0800 Subject: [PATCH 04/20] Return an error through yajl_gen_get_buf --- ext/yajl/api/yajl_common.h | 6 ++++++ ext/yajl/api/yajl_gen.h | 6 ++++-- ext/yajl/yajl_gen.c | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ext/yajl/api/yajl_common.h b/ext/yajl/api/yajl_common.h index dbd84e4b..8c9cbcde 100644 --- a/ext/yajl/api/yajl_common.h +++ b/ext/yajl/api/yajl_common.h @@ -56,6 +56,12 @@ extern "C" { # endif #endif +#if defined(__GNUC__) +#define YAJL_WARN_UNUSED __attribute__ ((warn_unused_result)) +#else +#define YAJL_WARN_UNUSED +#endif + /** pointer to a malloc function, supporting client overriding memory * allocation routines */ typedef void * (*yajl_malloc_func)(void *ctx, unsigned int sz); diff --git a/ext/yajl/api/yajl_gen.h b/ext/yajl/api/yajl_gen.h index a04ade3a..e084f5cc 100644 --- a/ext/yajl/api/yajl_gen.h +++ b/ext/yajl/api/yajl_gen.h @@ -65,7 +65,9 @@ extern "C" { * buffer to get from */ yajl_gen_no_buf, /** Tried to decrement at depth 0 */ - yajl_depth_underflow + yajl_depth_underflow, + /** Allocation error */ + yajl_gen_alloc_error } yajl_gen_status; /** an opaque handle to a generator */ @@ -148,7 +150,7 @@ extern "C" { /** access the null terminated generator buffer. If incrementally * outputing JSON, one should call yajl_gen_clear to clear the * buffer. This allows stream generation. */ - YAJL_API yajl_gen_status yajl_gen_get_buf(yajl_gen hand, + YAJL_API YAJL_WARN_UNUSED yajl_gen_status yajl_gen_get_buf(yajl_gen hand, const unsigned char ** buf, unsigned int * len); diff --git a/ext/yajl/yajl_gen.c b/ext/yajl/yajl_gen.c index b241a658..832b7710 100644 --- a/ext/yajl/yajl_gen.c +++ b/ext/yajl/yajl_gen.c @@ -362,6 +362,10 @@ yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf, unsigned int * len) { if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf; + yajl_buf_state buf_err = yajl_buf_err((yajl_buf)g->ctx); + if (buf_err) { + return yajl_gen_alloc_error; + } *buf = yajl_buf_data((yajl_buf)g->ctx); *len = yajl_buf_len((yajl_buf)g->ctx); return yajl_gen_status_ok; From e4eb8192ba21e7391f9a6c34bf1450259a11ad9a Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 16:14:12 -0800 Subject: [PATCH 05/20] Catch exit and raise alloc error in Ruby ext --- ext/yajl/yajl_ext.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index fe4c89f6..710a9e4b 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -93,7 +93,11 @@ static char *yajl_raise_encode_error_for_status(yajl_gen_status status, VALUE ob rb_raise(cEncodeError, "Invalid number: cannot encode Infinity, -Infinity, or NaN"); case yajl_gen_no_buf: rb_raise(cEncodeError, "YAJL internal error: yajl_gen_get_buf was called, but a print callback was specified, so no internal buffer is available"); + case yajl_gen_alloc_error: + rb_raise(cEncodeError, "YAJL internal error: failed to allocate memory"); default: + // fixme: why wasn't this already here?? + rb_raise(cEncodeError, "Encountered unknown YAJL status %d during JSON generation", status); return NULL; } } @@ -1135,6 +1139,7 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) { const unsigned char * buffer; unsigned int len; VALUE obj, io, blk, outBuff; + yajl_gen_status status; GetEncoder(self, wrapper); @@ -1148,7 +1153,11 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) { yajl_encode_part(wrapper, obj, io); /* just make sure we output the remaining buffer */ - yajl_gen_get_buf(wrapper->encoder, &buffer, &len); + status = yajl_gen_get_buf(wrapper->encoder, &buffer, &len); + if (status != yajl_gen_status_ok) { + yajl_raise_encode_error_for_status(status, obj); + } + outBuff = rb_str_new((const char *)buffer, len); #ifdef HAVE_RUBY_ENCODING_H rb_enc_associate(outBuff, utf8Encoding); From b67100be648088e9de2571674fac6956b0882df0 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 16:20:37 -0800 Subject: [PATCH 06/20] Replace assert(0) with rb_bug --- ext/yajl/yajl_ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index 710a9e4b..533683d4 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -921,7 +921,7 @@ static VALUE rb_yajl_projector_build_simple_value(yajl_event_stream_t parser, ya rb_raise(cParseError, "unexpected colon while constructing value"); default:; - assert(0); + rb_bug("we should never get here"); } } @@ -959,7 +959,7 @@ static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_eve } default:; { - assert(0); + rb_bug("we should never get here"); } } } From a11e8dc5acceb2ecc51eead683d1331463a57369 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 16:21:06 -0800 Subject: [PATCH 07/20] Fix compiler warnings --- ext/yajl/yajl_ext.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index 533683d4..fe5efa1c 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -193,14 +193,13 @@ static int yajl_encode_part_hash_i(VALUE key, VALUE val, VALUE iter_v) { if ((status = (call)) != yajl_gen_status_ok) { break; } void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) { - VALUE str, outBuff, otherObj; + VALUE str, outBuff; yajl_encoder_wrapper * w = wrapper; yajl_gen_status status; int idx = 0; const unsigned char * buffer; const char * cptr; unsigned int len; - VALUE *ptr; if (io != Qnil || w->on_progress_callback != Qnil) { status = yajl_gen_get_buf(w->encoder, &buffer, &len); From 538002b084d335ac261863c3b6b8cde52ad792ea Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Tue, 1 Mar 2022 17:02:00 -0800 Subject: [PATCH 08/20] Implement noop_allocs for buf_alloc_error So that we don't free the statically allocated error buffer --- ext/yajl/yajl_buf.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index cc8120af..23960328 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -35,6 +35,7 @@ #include #include #include +#include #define YAJL_BUF_INIT_SIZE 2048 @@ -46,9 +47,23 @@ struct yajl_buf_t { yajl_alloc_funcs * alloc; }; +static void *noop_realloc(void *ctx, void *ptr, unsigned int sz) { + fprintf(stderr, "Attempt to allocate on invalid yajl_buf_t\n"); + abort(); +} +static void *noop_malloc(void *ctx, unsigned int sz) { return noop_realloc(ctx, NULL, sz); } +static void noop_free(void *ctx, void *ptr) { } + +static yajl_alloc_funcs noop_allocs = { + .malloc = &noop_malloc, + .realloc = &noop_realloc, + .free = &noop_free, +}; + // A buffer to be returned if the initial allocation fails static struct yajl_buf_t buf_alloc_error = { - .state = yajl_buf_alloc_failed + .state = yajl_buf_alloc_failed, + .alloc = &noop_allocs }; #include From 09bc8604927cb10046249e0023d6fd1154ab404e Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 10:14:41 -0800 Subject: [PATCH 09/20] More asserts and add sanitize to extconf --- ext/yajl/extconf.rb | 5 +++++ ext/yajl/yajl_buf.c | 3 +++ 2 files changed, 8 insertions(+) diff --git a/ext/yajl/extconf.rb b/ext/yajl/extconf.rb index 1172e492..631c68c7 100644 --- a/ext/yajl/extconf.rb +++ b/ext/yajl/extconf.rb @@ -4,4 +4,9 @@ $CFLAGS << ' -Wall -funroll-loops -Wno-declaration-after-statement' $CFLAGS << ' -Werror-implicit-function-declaration -Wextra -O0 -ggdb3' if ENV['DEBUG'] +if ENV['SANITIZE'] + $CFLAGS << ' -fsanitize=address' + $LDFLAGS << ' -fsanitize=address' +end + create_makefile('yajl/yajl') diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index 23960328..d100ccdf 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -170,6 +170,8 @@ void yajl_buf_append(yajl_buf buf, const void * data, unsigned int len) void yajl_buf_clear(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); buf->used = 0; if (buf->data) buf->data[buf->used] = 0; } @@ -192,6 +194,7 @@ void yajl_buf_truncate(yajl_buf buf, unsigned int len) { assert(buf); + assert(!yajl_buf_err(buf)); assert(len <= buf->used); buf->used = len; } From bef7072fb64cd34776540d2e3052a984c59ad0bf Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 10:29:56 -0800 Subject: [PATCH 10/20] Check for failed buf allocations yajl_parser --- ext/yajl/api/yajl_parse.h | 4 +++- ext/yajl/yajl.c | 3 +++ ext/yajl/yajl_ext.c | 8 +++++++- ext/yajl/yajl_parser.c | 2 ++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ext/yajl/api/yajl_parse.h b/ext/yajl/api/yajl_parse.h index b65f54f7..09b3b51a 100644 --- a/ext/yajl/api/yajl_parse.h +++ b/ext/yajl/api/yajl_parse.h @@ -55,7 +55,9 @@ extern "C" { yajl_status_insufficient_data, /** An error occured during the parse. Call yajl_get_error for * more information about the encountered error */ - yajl_status_error + yajl_status_error, + /** an allocation failed */ + yajl_status_alloc_failed, } yajl_status; /** attain a human readable, english, string for an error */ diff --git a/ext/yajl/yajl.c b/ext/yajl/yajl.c index 50bca443..44113ab4 100644 --- a/ext/yajl/yajl.c +++ b/ext/yajl/yajl.c @@ -56,6 +56,9 @@ yajl_status_to_string(yajl_status stat) case yajl_status_error: statStr = "parse error"; break; + case yajl_status_alloc_failed: + statStr = "allocation failed"; + break; } return statStr; } diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index fe5efa1c..e0ba047e 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -314,11 +314,17 @@ void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle stat = yajl_parse(parser, chunk, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + if (stat == yajl_status_ok || stat == yajl_status_insufficient_data) { + // success + } else if (stat == yajl_status_error) { unsigned char * str = yajl_get_error(parser, 1, chunk, len); VALUE errobj = rb_exc_new2(cParseError, (const char*) str); yajl_free_error(parser, str); rb_exc_raise(errobj); + } else { + const char * str = yajl_status_to_string(stat); + VALUE errobj = rb_exc_new2(cParseError, (const char*) str); + rb_exc_raise(errobj); } } diff --git a/ext/yajl/yajl_parser.c b/ext/yajl/yajl_parser.c index 7cb88f7f..9b9741ee 100644 --- a/ext/yajl/yajl_parser.c +++ b/ext/yajl/yajl_parser.c @@ -185,6 +185,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->callbacks && hand->callbacks->yajl_string) { yajl_buf_clear(hand->decodeBuf); yajl_string_decode(hand->decodeBuf, buf, bufLen); + if (yajl_buf_err(hand->decodeBuf)) + return yajl_status_alloc_failed; _CC_CHK(hand->callbacks->yajl_string( hand->ctx, yajl_buf_data(hand->decodeBuf), yajl_buf_len(hand->decodeBuf))); From ce2f5e1224abbd8cf3e3f0a894483bd09b7eb502 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 12:05:12 -0800 Subject: [PATCH 11/20] Check for error on rb_yajl_projector_build_string --- ext/yajl/yajl_ext.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index e0ba047e..3eccfa21 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -949,6 +949,9 @@ static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_eve yajl_buf strBuf = yajl_buf_alloc(parser->funcs); yajl_string_decode(strBuf, (const unsigned char *)event.buf, event.len); + if (yajl_buf_err(strBuf)) { + rb_raise(cParseError, "YAJL internal error: failed to allocate memory"); + } VALUE str = rb_str_new((const char *)yajl_buf_data(strBuf), yajl_buf_len(strBuf)); rb_enc_associate(str, utf8Encoding); From d4c47af50fd0f6d3d03eacb02e81fc3024f3b094 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 11:17:06 -0800 Subject: [PATCH 12/20] Check buffer state inside lexer and parser --- ext/yajl/yajl_lex.c | 9 ++++++++- ext/yajl/yajl_lex.h | 3 ++- ext/yajl/yajl_parser.c | 11 +++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ext/yajl/yajl_lex.c b/ext/yajl/yajl_lex.c index 37114556..6fe74254 100644 --- a/ext/yajl/yajl_lex.c +++ b/ext/yajl/yajl_lex.c @@ -633,7 +633,12 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, lexer->bufInUse = 1; yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); lexer->bufOff = 0; - + + if (yajl_buf_err(lexer->buf)) { + lexer->error = yajl_lex_alloc_failed; + return yajl_tok_error; + } + if (tok != yajl_tok_eof) { *outBuf = yajl_buf_data(lexer->buf); *outLen = yajl_buf_len(lexer->buf); @@ -700,6 +705,8 @@ yajl_lex_error_to_string(yajl_lex_error error) case yajl_lex_unallowed_comment: return "probable comment found in input text, comments are " "not enabled."; + case yajl_lex_alloc_failed: + return "allocation failed"; } return "unknown error code"; } diff --git a/ext/yajl/yajl_lex.h b/ext/yajl/yajl_lex.h index ebda7672..f6dd3333 100644 --- a/ext/yajl/yajl_lex.h +++ b/ext/yajl/yajl_lex.h @@ -120,7 +120,8 @@ typedef enum { yajl_lex_missing_integer_after_decimal, yajl_lex_missing_integer_after_exponent, yajl_lex_missing_integer_after_minus, - yajl_lex_unallowed_comment + yajl_lex_unallowed_comment, + yajl_lex_alloc_failed } yajl_lex_error; YAJL_API diff --git a/ext/yajl/yajl_parser.c b/ext/yajl/yajl_parser.c index 9b9741ee..b27ee289 100644 --- a/ext/yajl/yajl_parser.c +++ b/ext/yajl/yajl_parser.c @@ -134,6 +134,11 @@ yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText, return yajl_status_client_canceled; \ } +/* check for buffer error */ +#define _BUF_CHK(x) \ + if (yajl_buf_err(x)) { \ + return yajl_status_alloc_failed; \ + } yajl_status yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, @@ -185,8 +190,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->callbacks && hand->callbacks->yajl_string) { yajl_buf_clear(hand->decodeBuf); yajl_string_decode(hand->decodeBuf, buf, bufLen); - if (yajl_buf_err(hand->decodeBuf)) - return yajl_status_alloc_failed; + _BUF_CHK(hand->decodeBuf); _CC_CHK(hand->callbacks->yajl_string( hand->ctx, yajl_buf_data(hand->decodeBuf), yajl_buf_len(hand->decodeBuf))); @@ -236,6 +240,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, long int i = 0; yajl_buf_clear(hand->decodeBuf); yajl_buf_append(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); i = strtol((const char *) buf, NULL, 10); if ((i == LONG_MIN || i == LONG_MAX) && @@ -263,6 +268,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, double d = 0.0; yajl_buf_clear(hand->decodeBuf); yajl_buf_append(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); d = strtod((char *) buf, NULL); if ((d == HUGE_VAL || d == -HUGE_VAL) && @@ -344,6 +350,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->callbacks && hand->callbacks->yajl_map_key) { yajl_buf_clear(hand->decodeBuf); yajl_string_decode(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); bufLen = yajl_buf_len(hand->decodeBuf); } From 7ab2f323f0660552092d4742840efa98067a9b69 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 12:55:21 -0800 Subject: [PATCH 13/20] Rewrite yajl_bs_push as inline function --- ext/yajl/yajl_bytestack.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/yajl/yajl_bytestack.h b/ext/yajl/yajl_bytestack.h index 872ede11..c84be814 100644 --- a/ext/yajl/yajl_bytestack.h +++ b/ext/yajl/yajl_bytestack.h @@ -66,15 +66,16 @@ typedef struct yajl_bytestack_t #define yajl_bs_current(obs) \ (assert((obs).used > 0), (obs).stack[(obs).used - 1]) -#define yajl_bs_push(obs, byte) { \ - if (((obs).size - (obs).used) == 0) { \ - (obs).size += YAJL_BS_INC; \ - (obs).stack = (obs).yaf->realloc((obs).yaf->ctx,\ - (void *) (obs).stack, (obs).size);\ - } \ - (obs).stack[((obs).used)++] = (byte); \ +static inline void yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { + if ((obs->size - obs->used) == 0) { + obs->size += YAJL_BS_INC; + obs->stack = obs->yaf->realloc(obs->yaf->ctx, (void *)obs->stack, obs->size); + } + obs->stack[obs->used++] = byte; } - + +#define yajl_bs_push(obs, byte) yajl_bs_push_inline(&(obs), (byte)) + /* removes the top item of the stack, returns nothing */ #define yajl_bs_pop(obs) { ((obs).used)--; } From 7dc47b131a8db65bb547f2fe43e1939bbf29f126 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 12:59:59 -0800 Subject: [PATCH 14/20] Check for allocation errors in yajl_bs_push --- ext/yajl/yajl_bytestack.h | 10 +++++++++- ext/yajl/yajl_parser.c | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ext/yajl/yajl_bytestack.h b/ext/yajl/yajl_bytestack.h index c84be814..8040bf1a 100644 --- a/ext/yajl/yajl_bytestack.h +++ b/ext/yajl/yajl_bytestack.h @@ -38,9 +38,11 @@ #ifndef __YAJL_BYTESTACK_H__ #define __YAJL_BYTESTACK_H__ +#include #include "api/yajl_common.h" #define YAJL_BS_INC 128 +#define YAJL_BS_MAX_SIZE UINT_MAX typedef struct yajl_bytestack_t { @@ -66,12 +68,18 @@ typedef struct yajl_bytestack_t #define yajl_bs_current(obs) \ (assert((obs).used > 0), (obs).stack[(obs).used - 1]) -static inline void yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { +/* 0: success, 1: error */ +static inline int yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { if ((obs->size - obs->used) == 0) { + if (obs->size > YAJL_BS_MAX_SIZE - YAJL_BS_INC) + return 1; obs->size += YAJL_BS_INC; obs->stack = obs->yaf->realloc(obs->yaf->ctx, (void *)obs->stack, obs->size); + if (!obs->stack) + return 1; } obs->stack[obs->used++] = byte; + return 0; } #define yajl_bs_push(obs, byte) yajl_bs_push_inline(&(obs), (byte)) diff --git a/ext/yajl/yajl_parser.c b/ext/yajl/yajl_parser.c index b27ee289..3065a969 100644 --- a/ext/yajl/yajl_parser.c +++ b/ext/yajl/yajl_parser.c @@ -328,7 +328,9 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, } } if (stateToPush != yajl_state_start) { - yajl_bs_push(hand->stateStack, stateToPush); + if (yajl_bs_push(hand->stateStack, stateToPush)) { + return yajl_status_alloc_failed; + } } goto around_again; From a7f8ebb84d3ea74f50a7f3946a7dfb88e540a46d Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 13:13:33 -0800 Subject: [PATCH 15/20] Return on error from yajl_bs_push_inline --- ext/yajl/yajl.c | 10 +++++++--- ext/yajl/yajl_bytestack.h | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/ext/yajl/yajl.c b/ext/yajl/yajl.c index 44113ab4..5a5380c6 100644 --- a/ext/yajl/yajl.c +++ b/ext/yajl/yajl.c @@ -86,6 +86,8 @@ yajl_alloc(const yajl_callbacks * callbacks, } hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t)); + if (hand == NULL) + return NULL; /* copy in pointers to allocation routines */ memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs)); @@ -97,12 +99,14 @@ yajl_alloc(const yajl_callbacks * callbacks, hand->callbacks = callbacks; hand->ctx = ctx; - hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8); + hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8); // fixme: check allocation hand->bytesConsumed = 0; - hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); + hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); // fixme: check allocation yajl_bs_init(hand->stateStack, &(hand->alloc)); - yajl_bs_push(hand->stateStack, yajl_state_start); + if (yajl_bs_push(hand->stateStack, yajl_state_start)) { + return NULL; + } return hand; } diff --git a/ext/yajl/yajl_bytestack.h b/ext/yajl/yajl_bytestack.h index 8040bf1a..f2f14a3a 100644 --- a/ext/yajl/yajl_bytestack.h +++ b/ext/yajl/yajl_bytestack.h @@ -39,6 +39,7 @@ #define __YAJL_BYTESTACK_H__ #include +#include #include "api/yajl_common.h" #define YAJL_BS_INC 128 @@ -69,7 +70,8 @@ typedef struct yajl_bytestack_t (assert((obs).used > 0), (obs).stack[(obs).used - 1]) /* 0: success, 1: error */ -static inline int yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { +static inline YAJL_WARN_UNUSED +int yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { if ((obs->size - obs->used) == 0) { if (obs->size > YAJL_BS_MAX_SIZE - YAJL_BS_INC) return 1; @@ -87,8 +89,14 @@ static inline int yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { /* removes the top item of the stack, returns nothing */ #define yajl_bs_pop(obs) { ((obs).used)--; } -#define yajl_bs_set(obs, byte) \ - (obs).stack[((obs).used) - 1] = (byte); - +static inline +void +yajl_bs_set_inline(yajl_bytestack *obs, unsigned char byte) { + assert(obs->used > 0); + assert(obs->size >= obs->used); + obs->stack[obs->used - 1] = byte; +} + +#define yajl_bs_set(obs, byte) yajl_bs_set_inline(&obs, byte) #endif From c2b52b349069ea3013dd030d2054a51494fbf005 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 13:22:48 -0800 Subject: [PATCH 16/20] Add assertions --- ext/yajl/yajl.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/yajl/yajl.c b/ext/yajl/yajl.c index 5a5380c6..c15af643 100644 --- a/ext/yajl/yajl.c +++ b/ext/yajl/yajl.c @@ -113,12 +113,14 @@ yajl_alloc(const yajl_callbacks * callbacks, void yajl_reset_parser(yajl_handle hand) { + assert(hand); hand->lexer = yajl_lex_realloc(hand->lexer); } void yajl_free(yajl_handle handle) { + assert(handle); yajl_bs_free(handle->stateStack); yajl_buf_free(handle->decodeBuf); yajl_lex_free(handle->lexer); @@ -129,6 +131,7 @@ yajl_status yajl_parse(yajl_handle hand, const unsigned char * jsonText, unsigned int jsonTextLen) { + assert(hand); yajl_status status; status = yajl_do_parse(hand, jsonText, jsonTextLen); return status; @@ -137,6 +140,7 @@ yajl_parse(yajl_handle hand, const unsigned char * jsonText, yajl_status yajl_parse_complete(yajl_handle hand) { + assert(hand); /* The particular case we want to handle is a trailing number. * Further input consisting of digits could cause our interpretation * of the number to change (buffered "1" but "2" comes in). @@ -150,6 +154,7 @@ unsigned char * yajl_get_error(yajl_handle hand, int verbose, const unsigned char * jsonText, unsigned int jsonTextLen) { + assert(hand); return yajl_render_error_string(hand, jsonText, jsonTextLen, verbose); } From 1d01cae5d018b045e6a71d92b72eb041d6db7872 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 13:44:36 -0800 Subject: [PATCH 17/20] record error in BUF_CHK --- ext/yajl/yajl_parser.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ext/yajl/yajl_parser.c b/ext/yajl/yajl_parser.c index 3065a969..03e52bec 100644 --- a/ext/yajl/yajl_parser.c +++ b/ext/yajl/yajl_parser.c @@ -135,9 +135,12 @@ yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText, } /* check for buffer error */ -#define _BUF_CHK(x) \ - if (yajl_buf_err(x)) { \ - return yajl_status_alloc_failed; \ +#define _BUF_CHK(x) \ + if (yajl_buf_err(x)) { \ + yajl_bs_set(hand->stateStack, yajl_state_parse_error); \ + hand->parseError = \ + "allocation failed"; \ + return yajl_status_alloc_failed; \ } yajl_status From 36410d536b676e836637bb20574a56ebc920eb83 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 2 Mar 2022 14:17:59 -0800 Subject: [PATCH 18/20] Check need < buf->used We're guaranteed a power of 2 so that this becomes 0, but we might as well use a check for overflow that works in more cases. Unsigned integer overflow is defined behaviour, so this should be safe. --- ext/yajl/yajl_buf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index d100ccdf..7c76793c 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -118,8 +118,8 @@ yajl_buf_state yajl_buf_ensure_available(yajl_buf buf, unsigned int want) while (want >= (need - buf->used)) need <<= 1; - // overflow - if (need == 0) { + // Check for overflow + if (need < buf->used) { return yajl_buf_set_error(buf, yajl_buf_overflow); } From b99748f9483c048cf2c857f4c34f0d555c9d6434 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 3 Mar 2022 09:42:29 -0800 Subject: [PATCH 19/20] Check for malloc failure in yajl_lex_alloc --- ext/yajl/yajl.c | 8 ++++++-- ext/yajl/yajl_lex.c | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ext/yajl/yajl.c b/ext/yajl/yajl.c index c15af643..0f8b03d4 100644 --- a/ext/yajl/yajl.c +++ b/ext/yajl/yajl.c @@ -99,9 +99,13 @@ yajl_alloc(const yajl_callbacks * callbacks, hand->callbacks = callbacks; hand->ctx = ctx; - hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8); // fixme: check allocation + hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8); + if (!hand->lexer) { + YA_FREE(afs, hand); + return NULL; + } hand->bytesConsumed = 0; - hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); // fixme: check allocation + hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); yajl_bs_init(hand->stateStack, &(hand->alloc)); if (yajl_bs_push(hand->stateStack, yajl_state_start)) { diff --git a/ext/yajl/yajl_lex.c b/ext/yajl/yajl_lex.c index 6fe74254..038a3b15 100644 --- a/ext/yajl/yajl_lex.c +++ b/ext/yajl/yajl_lex.c @@ -118,6 +118,8 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc, unsigned int allowComments, unsigned int validateUTF8) { yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t)); + if (!lxr) + return NULL; memset((void *) lxr, 0, sizeof(struct yajl_lexer_t)); lxr->buf = yajl_buf_alloc(alloc); lxr->allowComments = allowComments; From 15093d3f6ed1250a8965ec60a6b633c8792cb2f4 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Mon, 4 Apr 2022 15:20:25 -0700 Subject: [PATCH 20/20] Version 1.4.2 --- lib/yajl/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/yajl/version.rb b/lib/yajl/version.rb index a3203f2b..5378ace6 100644 --- a/lib/yajl/version.rb +++ b/lib/yajl/version.rb @@ -1,3 +1,3 @@ module Yajl - VERSION = '1.4.0' + VERSION = '1.4.2' end