diff --git a/ext/yajl/api/yajl_common.h b/ext/yajl/api/yajl_common.h index dbd84e4b..8c9cbcde 100644 --- a/ext/yajl/api/yajl_common.h +++ b/ext/yajl/api/yajl_common.h @@ -56,6 +56,12 @@ extern "C" { # endif #endif +#if defined(__GNUC__) +#define YAJL_WARN_UNUSED __attribute__ ((warn_unused_result)) +#else +#define YAJL_WARN_UNUSED +#endif + /** pointer to a malloc function, supporting client overriding memory * allocation routines */ typedef void * (*yajl_malloc_func)(void *ctx, unsigned int sz); diff --git a/ext/yajl/api/yajl_gen.h b/ext/yajl/api/yajl_gen.h index a04ade3a..e084f5cc 100644 --- a/ext/yajl/api/yajl_gen.h +++ b/ext/yajl/api/yajl_gen.h @@ -65,7 +65,9 @@ extern "C" { * buffer to get from */ yajl_gen_no_buf, /** Tried to decrement at depth 0 */ - yajl_depth_underflow + yajl_depth_underflow, + /** Allocation error */ + yajl_gen_alloc_error } yajl_gen_status; /** an opaque handle to a generator */ @@ -148,7 +150,7 @@ extern "C" { /** access the null terminated generator buffer. If incrementally * outputing JSON, one should call yajl_gen_clear to clear the * buffer. This allows stream generation. */ - YAJL_API yajl_gen_status yajl_gen_get_buf(yajl_gen hand, + YAJL_API YAJL_WARN_UNUSED yajl_gen_status yajl_gen_get_buf(yajl_gen hand, const unsigned char ** buf, unsigned int * len); diff --git a/ext/yajl/api/yajl_parse.h b/ext/yajl/api/yajl_parse.h index b65f54f7..09b3b51a 100644 --- a/ext/yajl/api/yajl_parse.h +++ b/ext/yajl/api/yajl_parse.h @@ -55,7 +55,9 @@ extern "C" { yajl_status_insufficient_data, /** An error occured during the parse. Call yajl_get_error for * more information about the encountered error */ - yajl_status_error + yajl_status_error, + /** an allocation failed */ + yajl_status_alloc_failed, } yajl_status; /** attain a human readable, english, string for an error */ diff --git a/ext/yajl/extconf.rb b/ext/yajl/extconf.rb index 1172e492..631c68c7 100644 --- a/ext/yajl/extconf.rb +++ b/ext/yajl/extconf.rb @@ -4,4 +4,9 @@ $CFLAGS << ' -Wall -funroll-loops -Wno-declaration-after-statement' $CFLAGS << ' -Werror-implicit-function-declaration -Wextra -O0 -ggdb3' if ENV['DEBUG'] +if ENV['SANITIZE'] + $CFLAGS << ' -fsanitize=address' + $LDFLAGS << ' -fsanitize=address' +end + create_makefile('yajl/yajl') diff --git a/ext/yajl/yajl.c b/ext/yajl/yajl.c index 50bca443..0f8b03d4 100644 --- a/ext/yajl/yajl.c +++ b/ext/yajl/yajl.c @@ -56,6 +56,9 @@ yajl_status_to_string(yajl_status stat) case yajl_status_error: statStr = "parse error"; break; + case yajl_status_alloc_failed: + statStr = "allocation failed"; + break; } return statStr; } @@ -83,6 +86,8 @@ yajl_alloc(const yajl_callbacks * callbacks, } hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t)); + if (hand == NULL) + return NULL; /* copy in pointers to allocation routines */ memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs)); @@ -95,23 +100,31 @@ yajl_alloc(const yajl_callbacks * callbacks, hand->callbacks = callbacks; hand->ctx = ctx; hand->lexer = yajl_lex_alloc(&(hand->alloc), allowComments, validateUTF8); + if (!hand->lexer) { + YA_FREE(afs, hand); + return NULL; + } hand->bytesConsumed = 0; hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); yajl_bs_init(hand->stateStack, &(hand->alloc)); - yajl_bs_push(hand->stateStack, yajl_state_start); + if (yajl_bs_push(hand->stateStack, yajl_state_start)) { + return NULL; + } return hand; } void yajl_reset_parser(yajl_handle hand) { + assert(hand); hand->lexer = yajl_lex_realloc(hand->lexer); } void yajl_free(yajl_handle handle) { + assert(handle); yajl_bs_free(handle->stateStack); yajl_buf_free(handle->decodeBuf); yajl_lex_free(handle->lexer); @@ -122,6 +135,7 @@ yajl_status yajl_parse(yajl_handle hand, const unsigned char * jsonText, unsigned int jsonTextLen) { + assert(hand); yajl_status status; status = yajl_do_parse(hand, jsonText, jsonTextLen); return status; @@ -130,6 +144,7 @@ yajl_parse(yajl_handle hand, const unsigned char * jsonText, yajl_status yajl_parse_complete(yajl_handle hand) { + assert(hand); /* The particular case we want to handle is a trailing number. * Further input consisting of digits could cause our interpretation * of the number to change (buffered "1" but "2" comes in). @@ -143,6 +158,7 @@ unsigned char * yajl_get_error(yajl_handle hand, int verbose, const unsigned char * jsonText, unsigned int jsonTextLen) { + assert(hand); return yajl_render_error_string(hand, jsonText, jsonTextLen, verbose); } diff --git a/ext/yajl/yajl_buf.c b/ext/yajl/yajl_buf.c index 04e608a3..7c76793c 100644 --- a/ext/yajl/yajl_buf.c +++ b/ext/yajl/yajl_buf.c @@ -35,43 +35,114 @@ #include #include #include +#include #define YAJL_BUF_INIT_SIZE 2048 struct yajl_buf_t { + yajl_buf_state state; unsigned int len; unsigned int used; unsigned char * data; yajl_alloc_funcs * alloc; }; +static void *noop_realloc(void *ctx, void *ptr, unsigned int sz) { + fprintf(stderr, "Attempt to allocate on invalid yajl_buf_t\n"); + abort(); +} +static void *noop_malloc(void *ctx, unsigned int sz) { return noop_realloc(ctx, NULL, sz); } +static void noop_free(void *ctx, void *ptr) { } + +static yajl_alloc_funcs noop_allocs = { + .malloc = &noop_malloc, + .realloc = &noop_realloc, + .free = &noop_free, +}; + +// A buffer to be returned if the initial allocation fails +static struct yajl_buf_t buf_alloc_error = { + .state = yajl_buf_alloc_failed, + .alloc = &noop_allocs +}; + +#include + +yajl_buf_state yajl_buf_err(yajl_buf buf) +{ + assert(buf); + return buf->state; +} + +static +yajl_buf_state yajl_buf_set_error(yajl_buf buf, yajl_buf_state err) +{ + buf->state = err; + + // free and clear all data from the buffer + YA_FREE(buf->alloc, buf->data); + buf->len = 0; + buf->data = 0; + buf->used = 0; + + return err; +} + static -void yajl_buf_ensure_available(yajl_buf buf, unsigned int want) +yajl_buf_state yajl_buf_ensure_available(yajl_buf buf, unsigned int want) { unsigned int need; assert(buf != NULL); + if (buf->state != yajl_buf_ok) { + return buf->state; + } + /* first call */ if (buf->data == NULL) { buf->len = YAJL_BUF_INIT_SIZE; buf->data = (unsigned char *) YA_MALLOC(buf->alloc, buf->len); + if (buf->data == NULL) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + buf->data[0] = 0; } + if (want == 0) { + return yajl_buf_ok; + } + need = buf->len; while (want >= (need - buf->used)) need <<= 1; + // Check for overflow + if (need < buf->used) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + if (need != buf->len) { buf->data = (unsigned char *) YA_REALLOC(buf->alloc, buf->data, need); + + if (buf->data == NULL) { + return yajl_buf_set_error(buf, yajl_buf_overflow); + } + buf->len = need; } + + return yajl_buf_ok; } yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc) { yajl_buf b = YA_MALLOC(alloc, sizeof(struct yajl_buf_t)); + if (b == NULL) { + return &buf_alloc_error; + } + memset((void *) b, 0, sizeof(struct yajl_buf_t)); b->alloc = alloc; return b; @@ -86,7 +157,9 @@ void yajl_buf_free(yajl_buf buf) void yajl_buf_append(yajl_buf buf, const void * data, unsigned int len) { - yajl_buf_ensure_available(buf, len); + if (yajl_buf_ensure_available(buf, len)) { + return; + } if (len > 0) { assert(data != NULL); memcpy(buf->data + buf->used, data, len); @@ -97,23 +170,31 @@ void yajl_buf_append(yajl_buf buf, const void * data, unsigned int len) void yajl_buf_clear(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); buf->used = 0; if (buf->data) buf->data[buf->used] = 0; } const unsigned char * yajl_buf_data(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); return buf->data; } unsigned int yajl_buf_len(yajl_buf buf) { + assert(buf); + assert(!yajl_buf_err(buf)); return buf->used; } void yajl_buf_truncate(yajl_buf buf, unsigned int len) { + assert(buf); + assert(!yajl_buf_err(buf)); assert(len <= buf->used); buf->used = len; } diff --git a/ext/yajl/yajl_buf.h b/ext/yajl/yajl_buf.h index f97355b9..f78f212f 100644 --- a/ext/yajl/yajl_buf.h +++ b/ext/yajl/yajl_buf.h @@ -43,6 +43,12 @@ * call overhead. YMMV. */ +typedef enum { + yajl_buf_ok = 0, + yajl_buf_alloc_failed, + yajl_buf_overflow +} yajl_buf_state; + /** * yajl_buf is a buffer with exponential growth. the buffer ensures that * you are always null padded. @@ -77,4 +83,7 @@ unsigned int yajl_buf_len(yajl_buf buf); YAJL_API void yajl_buf_truncate(yajl_buf buf, unsigned int len); +/* get the state of buffer */ +yajl_buf_state yajl_buf_err(yajl_buf buf); + #endif diff --git a/ext/yajl/yajl_bytestack.h b/ext/yajl/yajl_bytestack.h index 872ede11..f2f14a3a 100644 --- a/ext/yajl/yajl_bytestack.h +++ b/ext/yajl/yajl_bytestack.h @@ -38,9 +38,12 @@ #ifndef __YAJL_BYTESTACK_H__ #define __YAJL_BYTESTACK_H__ +#include +#include #include "api/yajl_common.h" #define YAJL_BS_INC 128 +#define YAJL_BS_MAX_SIZE UINT_MAX typedef struct yajl_bytestack_t { @@ -66,20 +69,34 @@ typedef struct yajl_bytestack_t #define yajl_bs_current(obs) \ (assert((obs).used > 0), (obs).stack[(obs).used - 1]) -#define yajl_bs_push(obs, byte) { \ - if (((obs).size - (obs).used) == 0) { \ - (obs).size += YAJL_BS_INC; \ - (obs).stack = (obs).yaf->realloc((obs).yaf->ctx,\ - (void *) (obs).stack, (obs).size);\ - } \ - (obs).stack[((obs).used)++] = (byte); \ +/* 0: success, 1: error */ +static inline YAJL_WARN_UNUSED +int yajl_bs_push_inline(yajl_bytestack *obs, unsigned char byte) { + if ((obs->size - obs->used) == 0) { + if (obs->size > YAJL_BS_MAX_SIZE - YAJL_BS_INC) + return 1; + obs->size += YAJL_BS_INC; + obs->stack = obs->yaf->realloc(obs->yaf->ctx, (void *)obs->stack, obs->size); + if (!obs->stack) + return 1; + } + obs->stack[obs->used++] = byte; + return 0; } - + +#define yajl_bs_push(obs, byte) yajl_bs_push_inline(&(obs), (byte)) + /* removes the top item of the stack, returns nothing */ #define yajl_bs_pop(obs) { ((obs).used)--; } -#define yajl_bs_set(obs, byte) \ - (obs).stack[((obs).used) - 1] = (byte); - +static inline +void +yajl_bs_set_inline(yajl_bytestack *obs, unsigned char byte) { + assert(obs->used > 0); + assert(obs->size >= obs->used); + obs->stack[obs->used - 1] = byte; +} + +#define yajl_bs_set(obs, byte) yajl_bs_set_inline(&obs, byte) #endif diff --git a/ext/yajl/yajl_ext.c b/ext/yajl/yajl_ext.c index fe4c89f6..3eccfa21 100644 --- a/ext/yajl/yajl_ext.c +++ b/ext/yajl/yajl_ext.c @@ -93,7 +93,11 @@ static char *yajl_raise_encode_error_for_status(yajl_gen_status status, VALUE ob rb_raise(cEncodeError, "Invalid number: cannot encode Infinity, -Infinity, or NaN"); case yajl_gen_no_buf: rb_raise(cEncodeError, "YAJL internal error: yajl_gen_get_buf was called, but a print callback was specified, so no internal buffer is available"); + case yajl_gen_alloc_error: + rb_raise(cEncodeError, "YAJL internal error: failed to allocate memory"); default: + // fixme: why wasn't this already here?? + rb_raise(cEncodeError, "Encountered unknown YAJL status %d during JSON generation", status); return NULL; } } @@ -189,14 +193,13 @@ static int yajl_encode_part_hash_i(VALUE key, VALUE val, VALUE iter_v) { if ((status = (call)) != yajl_gen_status_ok) { break; } void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) { - VALUE str, outBuff, otherObj; + VALUE str, outBuff; yajl_encoder_wrapper * w = wrapper; yajl_gen_status status; int idx = 0; const unsigned char * buffer; const char * cptr; unsigned int len; - VALUE *ptr; if (io != Qnil || w->on_progress_callback != Qnil) { status = yajl_gen_get_buf(w->encoder, &buffer, &len); @@ -311,11 +314,17 @@ void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle stat = yajl_parse(parser, chunk, len); - if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) { + if (stat == yajl_status_ok || stat == yajl_status_insufficient_data) { + // success + } else if (stat == yajl_status_error) { unsigned char * str = yajl_get_error(parser, 1, chunk, len); VALUE errobj = rb_exc_new2(cParseError, (const char*) str); yajl_free_error(parser, str); rb_exc_raise(errobj); + } else { + const char * str = yajl_status_to_string(stat); + VALUE errobj = rb_exc_new2(cParseError, (const char*) str); + rb_exc_raise(errobj); } } @@ -917,7 +926,7 @@ static VALUE rb_yajl_projector_build_simple_value(yajl_event_stream_t parser, ya rb_raise(cParseError, "unexpected colon while constructing value"); default:; - assert(0); + rb_bug("we should never get here"); } } @@ -940,6 +949,9 @@ static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_eve yajl_buf strBuf = yajl_buf_alloc(parser->funcs); yajl_string_decode(strBuf, (const unsigned char *)event.buf, event.len); + if (yajl_buf_err(strBuf)) { + rb_raise(cParseError, "YAJL internal error: failed to allocate memory"); + } VALUE str = rb_str_new((const char *)yajl_buf_data(strBuf), yajl_buf_len(strBuf)); rb_enc_associate(str, utf8Encoding); @@ -955,7 +967,7 @@ static VALUE rb_yajl_projector_build_string(yajl_event_stream_t parser, yajl_eve } default:; { - assert(0); + rb_bug("we should never get here"); } } } @@ -1135,6 +1147,7 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) { const unsigned char * buffer; unsigned int len; VALUE obj, io, blk, outBuff; + yajl_gen_status status; GetEncoder(self, wrapper); @@ -1148,7 +1161,11 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) { yajl_encode_part(wrapper, obj, io); /* just make sure we output the remaining buffer */ - yajl_gen_get_buf(wrapper->encoder, &buffer, &len); + status = yajl_gen_get_buf(wrapper->encoder, &buffer, &len); + if (status != yajl_gen_status_ok) { + yajl_raise_encode_error_for_status(status, obj); + } + outBuff = rb_str_new((const char *)buffer, len); #ifdef HAVE_RUBY_ENCODING_H rb_enc_associate(outBuff, utf8Encoding); diff --git a/ext/yajl/yajl_gen.c b/ext/yajl/yajl_gen.c index b241a658..832b7710 100644 --- a/ext/yajl/yajl_gen.c +++ b/ext/yajl/yajl_gen.c @@ -362,6 +362,10 @@ yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf, unsigned int * len) { if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf; + yajl_buf_state buf_err = yajl_buf_err((yajl_buf)g->ctx); + if (buf_err) { + return yajl_gen_alloc_error; + } *buf = yajl_buf_data((yajl_buf)g->ctx); *len = yajl_buf_len((yajl_buf)g->ctx); return yajl_gen_status_ok; diff --git a/ext/yajl/yajl_lex.c b/ext/yajl/yajl_lex.c index 37114556..038a3b15 100644 --- a/ext/yajl/yajl_lex.c +++ b/ext/yajl/yajl_lex.c @@ -118,6 +118,8 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc, unsigned int allowComments, unsigned int validateUTF8) { yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t)); + if (!lxr) + return NULL; memset((void *) lxr, 0, sizeof(struct yajl_lexer_t)); lxr->buf = yajl_buf_alloc(alloc); lxr->allowComments = allowComments; @@ -633,7 +635,12 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, lexer->bufInUse = 1; yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); lexer->bufOff = 0; - + + if (yajl_buf_err(lexer->buf)) { + lexer->error = yajl_lex_alloc_failed; + return yajl_tok_error; + } + if (tok != yajl_tok_eof) { *outBuf = yajl_buf_data(lexer->buf); *outLen = yajl_buf_len(lexer->buf); @@ -700,6 +707,8 @@ yajl_lex_error_to_string(yajl_lex_error error) case yajl_lex_unallowed_comment: return "probable comment found in input text, comments are " "not enabled."; + case yajl_lex_alloc_failed: + return "allocation failed"; } return "unknown error code"; } diff --git a/ext/yajl/yajl_lex.h b/ext/yajl/yajl_lex.h index ebda7672..f6dd3333 100644 --- a/ext/yajl/yajl_lex.h +++ b/ext/yajl/yajl_lex.h @@ -120,7 +120,8 @@ typedef enum { yajl_lex_missing_integer_after_decimal, yajl_lex_missing_integer_after_exponent, yajl_lex_missing_integer_after_minus, - yajl_lex_unallowed_comment + yajl_lex_unallowed_comment, + yajl_lex_alloc_failed } yajl_lex_error; YAJL_API diff --git a/ext/yajl/yajl_parser.c b/ext/yajl/yajl_parser.c index 7cb88f7f..03e52bec 100644 --- a/ext/yajl/yajl_parser.c +++ b/ext/yajl/yajl_parser.c @@ -134,6 +134,14 @@ yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText, return yajl_status_client_canceled; \ } +/* check for buffer error */ +#define _BUF_CHK(x) \ + if (yajl_buf_err(x)) { \ + yajl_bs_set(hand->stateStack, yajl_state_parse_error); \ + hand->parseError = \ + "allocation failed"; \ + return yajl_status_alloc_failed; \ + } yajl_status yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, @@ -185,6 +193,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->callbacks && hand->callbacks->yajl_string) { yajl_buf_clear(hand->decodeBuf); yajl_string_decode(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); _CC_CHK(hand->callbacks->yajl_string( hand->ctx, yajl_buf_data(hand->decodeBuf), yajl_buf_len(hand->decodeBuf))); @@ -234,6 +243,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, long int i = 0; yajl_buf_clear(hand->decodeBuf); yajl_buf_append(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); i = strtol((const char *) buf, NULL, 10); if ((i == LONG_MIN || i == LONG_MAX) && @@ -261,6 +271,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, double d = 0.0; yajl_buf_clear(hand->decodeBuf); yajl_buf_append(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); d = strtod((char *) buf, NULL); if ((d == HUGE_VAL || d == -HUGE_VAL) && @@ -320,7 +331,9 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, } } if (stateToPush != yajl_state_start) { - yajl_bs_push(hand->stateStack, stateToPush); + if (yajl_bs_push(hand->stateStack, stateToPush)) { + return yajl_status_alloc_failed; + } } goto around_again; @@ -342,6 +355,7 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->callbacks && hand->callbacks->yajl_map_key) { yajl_buf_clear(hand->decodeBuf); yajl_string_decode(hand->decodeBuf, buf, bufLen); + _BUF_CHK(hand->decodeBuf); buf = yajl_buf_data(hand->decodeBuf); bufLen = yajl_buf_len(hand->decodeBuf); } diff --git a/lib/yajl/version.rb b/lib/yajl/version.rb index a3203f2b..5378ace6 100644 --- a/lib/yajl/version.rb +++ b/lib/yajl/version.rb @@ -1,3 +1,3 @@ module Yajl - VERSION = '1.4.0' + VERSION = '1.4.2' end