fix: huge memory consumption within previews producing stage

Makefile

@@ -9,20 +9,26 @@ VERSION=0.0.1
 COMMIT_HASH=$(shell git rev-parse --short HEAD)
 BUILD_TIMESTAMP=$(shell date '+%Y-%m-%dT%H:%M:%S')
 LDFLAGS=-ldflags "-X 'main.Version=${VERSION}' -X 'main.CommitHash=${COMMIT_HASH}' -X 'main.BuildTimestamp=${BUILD_TIMESTAMP}' -X 'main.Mode=Production'"
+CGO_LDFLAGS=-ljemalloc
+CGO_CFLAGS=-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
 
 .PHONY: build
 build: ## build executables
 	echo "Building frontend assets"
 	make -C ./web build
 	mkdir -p out/
 	echo "Building executable"
-	GOARCH=amd64 GOOS=linux go build ${LDFLAGS} -o out/${BINARY_NAME}_linux_amd64 cmd/tinytune/tinytune.go
+	GOARCH=amd64 \
+        GOOS=linux \
+        CGO_CFLAGS="${CGO_CFLAGS}" \
+        CGO_LDFLAGS="${CGO_LDFLAGS}" \
+        go build ${LDFLAGS} -o out/${BINARY_NAME}_linux_amd64 cmd/tinytune/tinytune.go
 	chmod +x out/${BINARY_NAME}_linux_amd64
 	echo "Done"
 
 .PHONY: run
 run: ## run tinytune server
-	go run cmd/tinytune/tinytune.go ./test/
+	CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go run cmd/tinytune/tinytune.go ./test/
 
 .PHONY: watch
 watch: ## run tinytune server and frontend in hot-reload way
@@ -40,7 +46,7 @@ test: ## run server tests
 
 .PHONY: ubuntu
 ubuntu: ## Install deps for ubuntu (libvips, ffmpeg)
-	sudo apt install build-essential libvips pkg-config libvips-dev ffmpeg -y
+	sudo apt install build-essential libvips pkg-config libvips-dev libjemalloc-dev ffmpeg -y
 	npm i --prefix ./web
 
 .PHONY: coverage

internal/handlers.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"net/http/pprof"
 	"net/url"
 	"path/filepath"
 	"slices"
@@ -205,6 +206,11 @@ func (s Server) registerHandlers(silent bool) http.Handler {
 
 	staticHandler := http.StripPrefix("/static", http.FileServer(http.FS(s.getAssets())))
 	mux.Handle("GET /static/", chain.Then(staticHandler))
+	mux.Handle("GET /debug/pprof/", http.HandlerFunc(pprof.Index))
+	mux.Handle("GET /debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	mux.Handle("GET /debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	mux.Handle("GET /debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	mux.Handle("GET /debug/pprof/trace", http.HandlerFunc(pprof.Trace))
 
 	return mux
 }

internal/server.go

@@ -7,8 +7,6 @@ import (
 	"io/fs"
 	"net"
 	"net/http"
-	//nolint:gosec
-	_ "net/http/pprof"
 	"os"
 	"time"
 

pkg/index/builder.go

@@ -21,6 +21,7 @@ var (
 
 type PreviewGenerator interface {
 	Pull(item preview.Source) (preview.Data, error)
+	Close()
 }
 
 type indexBuilderParams struct {
@@ -67,6 +68,10 @@ func (ib *indexBuilder) run(ctx context.Context, r io.Reader) error {
 		return err
 	}
 
+	if ib.params.preview != nil {
+		ib.params.preview.Close()
+	}
+
 	if err := ib.loadTree(); err != nil {
 		return err
 	}

pkg/index/builder_test.go

@@ -122,12 +122,12 @@ func TestIndexBuilderClearRemovedFiles(t *testing.T) {
 
 	itemNextPreviewDataHash := "b9237e340eb886b5e437e4bc6fd19be612ff6653f3f733c3998ef20f320e6c20"
 
-	imagePrevew, err := index.PullPreview(image.ID)
+	imagePreview, err := index.PullPreview(image.ID)
 	require.NoError(err)
-	require.NotNil(imagePrevew)
+	require.NotNil(imagePreview)
 
 	h := sha256.New()
-	h.Write(imagePrevew)
+	h.Write(imagePreview)
 
 	require.Equal(itemNextPreviewDataHash, hex.EncodeToString(h.Sum(nil)))
 }

pkg/index/index_test.go

@@ -126,6 +126,8 @@ func (mock mockPreviewGenerator) Pull(_ preview.Source) (preview.Data, error) {
 	return mockPreviewData{data: mock.sampleData}, nil
 }
 
+func (mock mockPreviewGenerator) Close() {}
+
 func TestIndexFiles(t *testing.T) {
 	t.Parallel()
 

pkg/preview/image.go

@@ -3,8 +3,6 @@ package preview
 import (
 	"errors"
 	"fmt"
-	"log/slog"
-	"os"
 	"strings"
 
 	"github.com/davidbyttow/govips/v2/vips"
@@ -26,52 +24,16 @@ var (
 	ErrImageExport              = errors.New("failed export the image")
 )
 
-//nolint:gochecknoinits
-func init() {
-	os.Setenv("MALLOC_ARENA_MAX", "2")
-	vips.LoggingSettings(func(domain string, level vips.LogLevel, msg string) {
-		domainSlog := slog.String("source", domain)
-
-		switch level {
-		case vips.LogLevelCritical:
-		case vips.LogLevelError:
-			slog.Error(msg, domainSlog)
-		case vips.LogLevelDebug:
-			slog.Debug(msg, domainSlog)
-		case vips.LogLevelWarning:
-			slog.Warn(msg, domainSlog)
-		case vips.LogLevelInfo:
-		case vips.LogLevelMessage:
-		default:
-			slog.Info(msg, domainSlog)
-		}
-	}, vips.LogLevelError)
-	vips.Startup(&vips.Config{
-		MaxCacheFiles: 1,
-	})
-}
-
-func imagePreview(path string, size int64) (data, error) {
+func imagePreview(path string) (data, error) {
 	preview := data{}
-	params := &vips.ImportParams{}
-
-	// for big jpegs (>500kb), use shrink load
-	if size > 1<<19 {
-		params.JpegShrinkFactor.Set(jpegShrinkFactor)
-	}
 
-	image, err := vips.LoadImageFromFile(path, params)
+	image, err := vips.NewThumbnailFromFile(path, maxWidthHeight, maxWidthHeight, vips.InterestingAll)
 	if err != nil {
 		return preview, fmt.Errorf("%w: %w", ErrVipsLoadImage, err)
 	}
 	defer image.Close()
 
 	preview.width, preview.height = image.Width(), image.Height()
-
-	if err := downScale(image, imageDefault); err != nil {
-		return preview, fmt.Errorf("%w: %w", ErrImageDownScale, err)
-	}
-
 	preview.data, err = exportWebP(image)
 
 	return preview, err

pkg/preview/image_test.go

@@ -3,6 +3,7 @@ package preview
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -23,18 +24,18 @@ func TestPreviewImage(t *testing.T) {
 		{
 			Name:       ".jpg",
 			SourcePath: "../../test/img/image.jpg",
-			DataLength: 10332,
-			Width:      750,
-			Height:     1000,
-			Hash:       "182e0008de9bdd21a8af91ce6e6cbc57197e604886de71791133147f217c6fa0",
+			DataLength: 15260,
+			Width:      256,
+			Height:     341,
+			Hash:       "916594618fa51ee0084de8525d867a53e7e9f2a9b8544d8c14eb2c17a45c7632",
 		},
 		{
 			Name:       ".gif",
 			SourcePath: "../../test/sample_minions.gif",
-			DataLength: 7794,
-			Width:      400,
-			Height:     200,
-			Hash:       "846c829de9f1e4490e7025dcba99c903a7cb93289cd669b28790ffbc838e3847",
+			DataLength: 18700,
+			Width:      512,
+			Height:     256,
+			Hash:       "262882df79b599b6a719f9a5f9dc0ed7ca541ca6897c290f762cd795e6edcec8",
 		},
 	}
 
@@ -43,9 +44,14 @@ func TestPreviewImage(t *testing.T) {
 			t.Parallel()
 
 			assert := assert.New(t)
-			preview, err := imagePreview(testCase.SourcePath, 1<<10)
+			preview, err := imagePreview(testCase.SourcePath)
 			require.NoError(t, err)
-			assert.Len(preview.Data(), testCase.DataLength)
+			assert.Len(
+				preview.Data(),
+				testCase.DataLength,
+				fmt.Sprintf("expected = %d, actual = %d", testCase.DataLength, len(preview.Data())),
+			)
+
 			width, height := preview.Resolution()
 			assert.Equal(testCase.Width, width)
 			assert.Equal(testCase.Height, height)

pkg/preview/preview.go

@@ -6,6 +6,8 @@ import (
 	"log/slog"
 	"sync/atomic"
 	"time"
+
+	"github.com/davidbyttow/govips/v2/vips"
 )
 
 var (
@@ -32,6 +34,29 @@ type Previewer struct {
 	timeout       time.Duration
 }
 
+//nolint:gochecknoinits
+func init() {
+	vips.LoggingSettings(func(domain string, level vips.LogLevel, msg string) {
+		domainSlog := slog.String("source", domain)
+
+		switch level {
+		case vips.LogLevelCritical, vips.LogLevelError:
+			slog.Error(msg, domainSlog)
+		case vips.LogLevelDebug:
+			slog.Debug(msg, domainSlog)
+		case vips.LogLevelWarning:
+			slog.Warn(msg, domainSlog)
+		case vips.LogLevelMessage, vips.LogLevelInfo:
+			slog.Info(msg, domainSlog)
+		default:
+			slog.Info(msg, domainSlog)
+		}
+	}, vips.LogLevelError)
+	vips.Startup(&vips.Config{
+		MaxCacheFiles: 1,
+	})
+}
+
 func NewPreviewer(opts ...Option) (*Previewer, error) {
 	preview := &Previewer{
 		maxImages:     -1,
@@ -84,7 +109,7 @@ func (p Previewer) Pull(src Source) (Data, error) {
 	}
 
 	if toImage {
-		preview, err := imagePreview(src.Path(), src.Size())
+		preview, err := imagePreview(src.Path())
 		if err != nil {
 			return defaultPreview, fmt.Errorf("%w: %w", ErrImagePreview, err)
 		}
@@ -114,3 +139,7 @@ func (p Previewer) Pull(src Source) (Data, error) {
 
 	return defaultPreview, nil
 }
+
+func (p Previewer) Close() {
+	vips.Shutdown()
+}

pkg/preview/preview_test.go

@@ -51,7 +51,7 @@ func TestPreview(t *testing.T) {
 	}{
 		{
 			source:   mockSource{image: true, path: "../../test/img/image.jpg"},
-			dataHash: "182e0008de9bdd21a8af91ce6e6cbc57197e604886de71791133147f217c6fa0",
+			dataHash: "916594618fa51ee0084de8525d867a53e7e9f2a9b8544d8c14eb2c17a45c7632",
 		},
 		{
 			source:   mockSource{video: true, path: "../../test/sample.mp4"},