Improve chunked video download memory usage

msgconv/media.go

@@ -92,35 +92,36 @@ func downloadChunkedVideo(ctx context.Context, mime, url string, maxSize int64)
 		return nil, fmt.Errorf("unexpected status code %d for HEAD request", resp.StatusCode)
 	} else if resp.Header.Get("Accept-Ranges") != "bytes" {
 		return nil, fmt.Errorf("server does not support byte range requests")
+	} else if resp.ContentLength <= 0 {
+		return nil, fmt.Errorf("server didn't return media size")
 	} else if resp.ContentLength > maxSize {
 		return nil, fmt.Errorf("%w (%.2f MiB)", ErrTooLargeFile, float64(resp.ContentLength)/1024/1024)
 	}
 	log.Debug().Int64("content_length", resp.ContentLength).Msg("Found video size to download in chunks")
 
 	const chunkSize = 1 * 1024 * 1024
-	fullData := make([]byte, 0, maxSize)
+	fullData := make([]byte, resp.ContentLength)
 	for i := int64(0); i < resp.ContentLength; i += chunkSize {
 		end := i + chunkSize - 1
 		if end > resp.ContentLength {
 			end = resp.ContentLength - 1
 		}
 		byteRange := fmt.Sprintf("bytes=%d-%d", i, end)
 		log.Debug().Str("range", byteRange).Msg("Downloading chunk")
-		data, err := downloadMedia(ctx, mime, url, maxSize, byteRange, false)
+		_, err = downloadMedia(ctx, mime, url, maxSize, byteRange, false, fullData[i:end+1])
 		if err != nil {
 			return nil, fmt.Errorf("failed to download chunk %d-%d: %w", i, end, err)
 		}
-		fullData = append(fullData, data...)
 	}
 	log.Debug().Int("data_length", len(fullData)).Msg("Download complete")
 	return fullData, nil
 }
 
 func DownloadMedia(ctx context.Context, mime, url string, maxSize int64) ([]byte, error) {
-	return downloadMedia(ctx, mime, url, maxSize, "", true)
+	return downloadMedia(ctx, mime, url, maxSize, "", true, nil)
 }
 
-func downloadMedia(ctx context.Context, mime, url string, maxSize int64, byteRange string, switchToChunked bool) ([]byte, error) {
+func downloadMedia(ctx context.Context, mime, url string, maxSize int64, byteRange string, switchToChunked bool, readInto []byte) ([]byte, error) {
 	zerolog.Ctx(ctx).Trace().Str("url", url).Msg("Downloading media")
 	if BypassOnionForMedia {
 		url = strings.ReplaceAll(url, "facebookcooa4ldbat4g7iacswl3p2zrf5nuylvnhxn6kqolvojixwid.onion", "fbcdn.net")
@@ -154,7 +155,16 @@ func downloadMedia(ctx context.Context, mime, url string, maxSize int64, byteRan
 		return nil, fmt.Errorf("%w (%.2f MiB)", ErrTooLargeFile, float64(resp.ContentLength)/1024/1024)
 	}
 	zerolog.Ctx(ctx).Debug().Int64("content_length", resp.ContentLength).Msg("Got media response, reading data")
-	if respData, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+2)); err != nil {
+	if readInto != nil {
+		if resp.ContentLength != int64(len(readInto)) {
+			return nil, fmt.Errorf("buffer size (%d) does not match content length (%d)", len(readInto), resp.ContentLength)
+		}
+		_, err = io.ReadFull(resp.Body, readInto)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read response data into buffer: %w", err)
+		}
+		return readInto, nil
+	} else if respData, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+2)); err != nil {
 		return nil, fmt.Errorf("failed to read response data: %w", err)
 	} else if int64(len(respData)) > maxSize {
 		return nil, ErrTooLargeFile