Skip to content

Commit

Permalink
http: Support chunked request bodies
Browse files Browse the repository at this point in the history 
This is a temporary support for chunked request bodies by converting
to Content-Length. This allows for processing of such requests until
a more permanent solution is developed.

A new configuration option "chunked_transform" has been added to enable
this feature. The option can be set as follows:
  {
    "settings": {
      "chunked_transform": true
    }
  }
By default, this option is set to false, which retains the current
behaviour of rejecting chunked requests with a '411 Length Required'
status code.

Please note that this is an experimental implementation.

Reviewed-by: Andrew Clayton <a.clayton@nginx.com>
  • Loading branch information
@hongzhidao
hongzhidao committed Jun 20, 2024
1 parent f80a36a commit 64f4c78
Show file tree
Hide file tree
Showing 6 changed files with 191 additions and 38 deletions.
3 changes: 3 additions & 0 deletions src/nxt_conf_validation.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = {
}, {
.name = nxt_string("server_version"),
.type = NXT_CONF_VLDT_BOOLEAN,
}, {
.name = nxt_string("chunked_transform"),
.type = NXT_CONF_VLDT_BOOLEAN,
},

NXT_CONF_VLDT_END
Expand Down
173 changes: 135 additions & 38 deletions src/nxt_h1proto.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,12 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data)
if (field->value_length == 7
&& memcmp(field->value, "chunked", 7) == 0)
{
if (r->chunked_field != NULL) {
return NXT_HTTP_BAD_REQUEST;
}

te = NXT_HTTP_TE_CHUNKED;
r->chunked_field = field;

} else {
te = NXT_HTTP_TE_UNSUPPORTED;
Expand All @@ -856,23 +861,36 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
{
size_t size, body_length, body_buffer_size, body_rest;
ssize_t res;
nxt_buf_t *in, *b;
nxt_buf_t *in, *b, *out, *chunk;
nxt_conn_t *c;
nxt_h1proto_t *h1p;
nxt_socket_conf_t *skcf;
nxt_http_status_t status;

static const nxt_str_t tmp_name_pattern = nxt_string("/req-XXXXXXXX");

h1p = r->proto.h1;
skcf = r->conf->socket_conf;

nxt_debug(task, "h1p request body read %O te:%d",
r->content_length_n, h1p->transfer_encoding);

switch (h1p->transfer_encoding) {

case NXT_HTTP_TE_CHUNKED:
status = NXT_HTTP_LENGTH_REQUIRED;
goto error;
if (!skcf->chunked_transform) {
status = NXT_HTTP_LENGTH_REQUIRED;
goto error;
}

if (r->content_length != NULL || !nxt_h1p_is_http11(h1p)) {
status = NXT_HTTP_BAD_REQUEST;
goto error;
}

r->chunked = 1;
h1p->chunked_parse.mem_pool = r->mem_pool;
break;

case NXT_HTTP_TE_UNSUPPORTED:
status = NXT_HTTP_NOT_IMPLEMENTED;
Expand All @@ -883,19 +901,20 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)
break;
}

if (r->content_length_n == -1 || r->content_length_n == 0) {
if (!r->chunked &&
(r->content_length_n == -1 || r->content_length_n == 0))
{
goto ready;
}

body_length = (size_t) r->content_length_n;

body_buffer_size = nxt_min(r->conf->socket_conf->body_buffer_size,
body_length);
body_buffer_size = nxt_min(skcf->body_buffer_size, body_length);

if (body_length > body_buffer_size) {
nxt_str_t *tmp_path, tmp_name;

tmp_path = &r->conf->socket_conf->body_temp_path;
tmp_path = &skcf->body_temp_path;

tmp_name.length = tmp_path->length + tmp_name_pattern.length;

Expand Down Expand Up @@ -946,31 +965,69 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r)

r->body = b;

body_rest = body_length;
body_rest = r->chunked ? 1 : body_length;

in = h1p->conn->read;

size = nxt_buf_mem_used_size(&in->mem);

if (size != 0) {
size = nxt_min(size, body_length);

if (nxt_buf_is_file(b)) {
res = nxt_fd_write(b->file->fd, in->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
status = NXT_HTTP_INTERNAL_SERVER_ERROR;
goto error;
}
if (r->chunked) {
out = nxt_http_chunk_parse(task, &h1p->chunked_parse, in);

b->file_end += size;
if (h1p->chunked_parse.error) {
status = NXT_HTTP_INTERNAL_SERVER_ERROR;
goto error;
}

if (h1p->chunked_parse.chunk_error) {
status = NXT_HTTP_BAD_REQUEST;
goto error;
}

for (chunk = out; chunk != NULL; chunk = chunk->next) {
size = nxt_buf_mem_used_size(&chunk->mem);

res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
status = NXT_HTTP_INTERNAL_SERVER_ERROR;
goto error;
}

b->file_end += size;

if ((size_t) b->file_end > skcf->max_body_size) {
status = NXT_HTTP_PAYLOAD_TOO_LARGE;
goto error;
}
}

if (h1p->chunked_parse.last) {
body_rest = 0;
}

} else {
size = nxt_min(size, body_length);
res = nxt_fd_write(b->file->fd, in->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
status = NXT_HTTP_INTERNAL_SERVER_ERROR;
goto error;
}

b->file_end += size;

in->mem.pos += size;
body_rest -= size;
}

} else {
size = nxt_min(body_buffer_size, size);
b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size);
}

in->mem.pos += size;
body_rest -= size;
in->mem.pos += size;
body_rest -= size;
}
}

nxt_debug(task, "h1p body rest: %uz", body_rest);
Expand Down Expand Up @@ -1028,9 +1085,10 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
{
size_t size, body_rest;
ssize_t res;
nxt_buf_t *b;
nxt_buf_t *b, *out, *chunk;
nxt_conn_t *c;
nxt_h1proto_t *h1p;
nxt_socket_conf_t *skcf;
nxt_http_request_t *r;
nxt_event_engine_t *engine;

Expand All @@ -1040,38 +1098,77 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data)
nxt_debug(task, "h1p conn request body read");

r = h1p->request;
skcf = r->conf->socket_conf;

engine = task->thread->engine;

b = c->read;

if (nxt_buf_is_file(b)) {
body_rest = b->file->size - b->file_end;

size = nxt_buf_mem_used_size(&b->mem);
size = nxt_min(size, body_rest);
if (r->chunked) {
body_rest = 1;

res = nxt_fd_write(b->file->fd, b->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
nxt_h1p_request_error(task, h1p, r);
return;
}
out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b);

b->file_end += size;
body_rest -= res;
if (h1p->chunked_parse.error) {
nxt_h1p_request_error(task, h1p, r);
return;
}

b->mem.pos += size;
if (h1p->chunked_parse.chunk_error) {
nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST);
return;
}

if (b->mem.pos == b->mem.free) {
if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
b->mem.free = b->mem.start;
for (chunk = out; chunk != NULL; chunk = chunk->next) {
size = nxt_buf_mem_used_size(&chunk->mem);
res = nxt_fd_write(b->file->fd, chunk->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
nxt_h1p_request_error(task, h1p, r);
return;
}

} else {
/* This required to avoid reading next request. */
b->mem.free = b->mem.end - body_rest;
b->file_end += size;

if ((size_t) b->file_end > skcf->max_body_size) {
nxt_h1p_request_error(task, h1p, r);
return;
}
}

b->mem.pos = b->mem.free;
if (h1p->chunked_parse.last) {
body_rest = 0;
}

} else {
body_rest = b->file->size - b->file_end;

size = nxt_buf_mem_used_size(&b->mem);
size = nxt_min(size, body_rest);

res = nxt_fd_write(b->file->fd, b->mem.pos, size);
if (nxt_slow_path(res < (ssize_t) size)) {
nxt_h1p_request_error(task, h1p, r);
return;
}

b->file_end += size;
body_rest -= res;

b->mem.pos += size;

if (b->mem.pos == b->mem.free) {
if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) {
b->mem.free = b->mem.start;

} else {
/* This required to avoid reading next request. */
b->mem.free = b->mem.end - body_rest;
}

b->mem.pos = b->mem.free;
}
}

} else {
Expand Down
2 changes: 2 additions & 0 deletions src/nxt_http.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ struct nxt_http_request_s {
nxt_list_t *fields;
nxt_http_field_t *content_type;
nxt_http_field_t *content_length;
nxt_http_field_t *chunked_field;
nxt_http_field_t *cookie;
nxt_http_field_t *referer;
nxt_http_field_t *user_agent;
Expand Down Expand Up @@ -204,6 +205,7 @@ struct nxt_http_request_s {
uint8_t inconsistent; /* 1 bit */
uint8_t error; /* 1 bit */
uint8_t websocket_handshake; /* 1 bit */
uint8_t chunked; /* 1 bit */
};


Expand Down
43 changes: 43 additions & 0 deletions src/nxt_http_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,15 +540,58 @@ static const nxt_http_request_state_t nxt_http_request_body_state
};


static nxt_int_t
nxt_http_request_chunked_transform(nxt_http_request_t *r)
{
size_t size;
u_char *p, *end;
nxt_http_field_t *f;

r->chunked_field->skip = 1;

size = r->body->file_end;

f = nxt_list_zero_add(r->fields);
if (nxt_slow_path(f == NULL)) {
return NXT_ERROR;
}

nxt_http_field_name_set(f, "Content-Length");

p = nxt_mp_nget(r->mem_pool, NXT_OFF_T_LEN);
if (nxt_slow_path(p == NULL)) {
return NXT_ERROR;
}

f->value = p;
end = nxt_sprintf(p, p + NXT_OFF_T_LEN, "%uz", size);
f->value_length = end - p;

r->content_length = f;
r->content_length_n = size;

return NXT_OK;
}


static void
nxt_http_request_ready(nxt_task_t *task, void *obj, void *data)
{
nxt_int_t ret;
nxt_http_action_t *action;
nxt_http_request_t *r;

r = obj;
action = r->conf->socket_conf->action;

if (r->chunked) {
ret = nxt_http_request_chunked_transform(r);
if (nxt_slow_path(ret != NXT_OK)) {
nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR);
return;
}
}

nxt_http_request_action(task, r, action);
}

Expand Down
7 changes: 7 additions & 0 deletions src/nxt_router.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1575,6 +1575,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = {
NXT_CONF_MAP_INT8,
offsetof(nxt_socket_conf_t, server_version),
},

{
nxt_string("chunked_transform"),
NXT_CONF_MAP_INT8,
offsetof(nxt_socket_conf_t, chunked_transform),
},
};


Expand Down Expand Up @@ -1994,6 +2000,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
skcf->proxy_read_timeout = 30 * 1000;

skcf->server_version = 1;
skcf->chunked_transform = 0;

skcf->websocket_conf.max_frame_size = 1024 * 1024;
skcf->websocket_conf.read_timeout = 60 * 1000;
Expand Down
1 change: 1 addition & 0 deletions src/nxt_router.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ typedef struct {
uint8_t discard_unsafe_fields; /* 1 bit */

uint8_t server_version; /* 1 bit */
uint8_t chunked_transform; /* 1 bit */

nxt_http_forward_t *forwarded;
nxt_http_forward_t *client_ip;
Expand Down

0 comments on commit 64f4c78

Please sign in to comment.