fix: utils StreamResponseBody() memory use for large get requests

The StreamResponseBody() called ctx.Write() in a loop with a small
buffer in an attempt to stream data back to client. But the
ctx.Write() was just calling append buffer to the response instead
of streaming the data back to the client.

The correct way to stream the response back is to use
(ctx *fasthttp.RequestCtx).SetBodyStream() to set the body stream
reader, and the response will automatically get streamed back
using the reader. This will also call Close() on our body
since we are providing an io.ReadCloser.

Testing this should be done with single large get requests such as
aws s3api get-object --bucket bucket --key file /tmp/data
for very large objects. The testing shows significantly reduced
memory usage for large objects once the streaming is enabled.

Fixes #1082

Author: benmcclelland

s3api/controllers/base.go

@@ -644,7 +644,12 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
 	}
 
 	if res.Body != nil {
-		err := utils.StreamResponseBody(ctx, res.Body)
+		// -1 will stream response body until EOF if content length not set
+		contentLen := -1
+		if res.ContentLength != nil {
+			contentLen = int(*res.ContentLength)
+		}
+		err := utils.StreamResponseBody(ctx, res.Body, contentLen)
 		if err != nil {
 			SendResponse(ctx, nil,
 				&MetaOpts{

s3api/utils/utils.go

@@ -204,26 +204,10 @@ func SetResponseHeaders(ctx *fiber.Ctx, headers []CustomHeader) {
 }
 
 // Streams the response body by chunks
-func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser) error {
-	buf := make([]byte, 4096) // 4KB chunks
-	defer rdr.Close()
-	for {
-		n, err := rdr.Read(buf)
-		if n > 0 {
-			_, writeErr := ctx.Write(buf[:n])
-			if writeErr != nil {
-				return fmt.Errorf("write chunk: %w", writeErr)
-			}
-		}
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-
-			return fmt.Errorf("read chunk: %w", err)
-		}
-	}
-
+func StreamResponseBody(ctx *fiber.Ctx, rdr io.ReadCloser, bodysize int) error {
+	// SetBodyStream will call Close() on the reader when the stream is done
+	// since rdr is a ReadCloser
+	ctx.Context().SetBodyStream(rdr, bodysize)
 	return nil
 }