diff --git a/Makefile b/Makefile index 2fefa40..9ee2bbf 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ VERSION=0.0.1 COMMIT_HASH=$(shell git rev-parse --short HEAD) BUILD_TIMESTAMP=$(shell date '+%Y-%m-%dT%H:%M:%S') LDFLAGS=-ldflags "-X 'main.Version=${VERSION}' -X 'main.CommitHash=${COMMIT_HASH}' -X 'main.BuildTimestamp=${BUILD_TIMESTAMP}' -X 'main.Mode=Production'" +CGO_LDFLAGS=-ljemalloc +CGO_CFLAGS=-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free .PHONY: build build: ## build executables @@ -16,13 +18,17 @@ build: ## build executables make -C ./web build mkdir -p out/ echo "Building executable" - GOARCH=amd64 GOOS=linux go build ${LDFLAGS} -o out/${BINARY_NAME}_linux_amd64 cmd/tinytune/tinytune.go + GOARCH=amd64 \ + GOOS=linux \ + CGO_CFLAGS="${CGO_CFLAGS}" \ + CGO_LDFLAGS="${CGO_LDFLAGS}" \ + go build ${LDFLAGS} -o out/${BINARY_NAME}_linux_amd64 cmd/tinytune/tinytune.go chmod +x out/${BINARY_NAME}_linux_amd64 echo "Done" .PHONY: run run: ## run tinytune server - go run cmd/tinytune/tinytune.go ./test/ + CGO_CFLAGS="${CGO_CFLAGS}" CGO_LDFLAGS="${CGO_LDFLAGS}" go run cmd/tinytune/tinytune.go ./test/ .PHONY: watch watch: ## run tinytune server and frontend in hot-reload way @@ -40,7 +46,7 @@ test: ## run server tests .PHONY: ubuntu ubuntu: ## Install deps for ubuntu (libvips, ffmpeg) - sudo apt install build-essential libvips pkg-config libvips-dev ffmpeg -y + sudo apt install build-essential libvips pkg-config libvips-dev libjemalloc-dev ffmpeg -y npm i --prefix ./web .PHONY: coverage diff --git a/internal/handlers.go b/internal/handlers.go index a0b6e42..04ac83f 100644 --- a/internal/handlers.go +++ b/internal/handlers.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "net/http" + "net/http/pprof" "net/url" "path/filepath" "slices" @@ -205,6 +206,11 @@ func (s Server) registerHandlers(silent bool) http.Handler { staticHandler := http.StripPrefix("/static", http.FileServer(http.FS(s.getAssets()))) mux.Handle("GET /static/", chain.Then(staticHandler)) + mux.Handle("GET /debug/pprof/", http.HandlerFunc(pprof.Index)) + mux.Handle("GET /debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline)) + mux.Handle("GET /debug/pprof/profile", http.HandlerFunc(pprof.Profile)) + mux.Handle("GET /debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) + mux.Handle("GET /debug/pprof/trace", http.HandlerFunc(pprof.Trace)) return mux } diff --git a/internal/server.go b/internal/server.go index f42a6ca..598cc12 100644 --- a/internal/server.go +++ b/internal/server.go @@ -7,8 +7,6 @@ import ( "io/fs" "net" "net/http" - //nolint:gosec - _ "net/http/pprof" "os" "time" diff --git a/pkg/index/builder.go b/pkg/index/builder.go index acab8a5..0650d47 100644 --- a/pkg/index/builder.go +++ b/pkg/index/builder.go @@ -21,6 +21,7 @@ var ( type PreviewGenerator interface { Pull(item preview.Source) (preview.Data, error) + Close() } type indexBuilderParams struct { @@ -67,6 +68,10 @@ func (ib *indexBuilder) run(ctx context.Context, r io.Reader) error { return err } + if ib.params.preview != nil { + ib.params.preview.Close() + } + if err := ib.loadTree(); err != nil { return err } diff --git a/pkg/index/builder_test.go b/pkg/index/builder_test.go index b4458d6..be4ce48 100644 --- a/pkg/index/builder_test.go +++ b/pkg/index/builder_test.go @@ -122,12 +122,12 @@ func TestIndexBuilderClearRemovedFiles(t *testing.T) { itemNextPreviewDataHash := "b9237e340eb886b5e437e4bc6fd19be612ff6653f3f733c3998ef20f320e6c20" - imagePrevew, err := index.PullPreview(image.ID) + imagePreview, err := index.PullPreview(image.ID) require.NoError(err) - require.NotNil(imagePrevew) + require.NotNil(imagePreview) h := sha256.New() - h.Write(imagePrevew) + h.Write(imagePreview) require.Equal(itemNextPreviewDataHash, hex.EncodeToString(h.Sum(nil))) } diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go index 7c33f98..8633e9a 100644 --- a/pkg/index/index_test.go +++ b/pkg/index/index_test.go @@ -126,6 +126,8 @@ func (mock mockPreviewGenerator) Pull(_ preview.Source) (preview.Data, error) { return mockPreviewData{data: mock.sampleData}, nil } +func (mock mockPreviewGenerator) Close() {} + func TestIndexFiles(t *testing.T) { t.Parallel() diff --git a/pkg/preview/image.go b/pkg/preview/image.go index 897132b..6901169 100644 --- a/pkg/preview/image.go +++ b/pkg/preview/image.go @@ -3,8 +3,6 @@ package preview import ( "errors" "fmt" - "log/slog" - "os" "strings" "github.com/davidbyttow/govips/v2/vips" @@ -26,52 +24,16 @@ var ( ErrImageExport = errors.New("failed export the image") ) -//nolint:gochecknoinits -func init() { - os.Setenv("MALLOC_ARENA_MAX", "2") - vips.LoggingSettings(func(domain string, level vips.LogLevel, msg string) { - domainSlog := slog.String("source", domain) - - switch level { - case vips.LogLevelCritical: - case vips.LogLevelError: - slog.Error(msg, domainSlog) - case vips.LogLevelDebug: - slog.Debug(msg, domainSlog) - case vips.LogLevelWarning: - slog.Warn(msg, domainSlog) - case vips.LogLevelInfo: - case vips.LogLevelMessage: - default: - slog.Info(msg, domainSlog) - } - }, vips.LogLevelError) - vips.Startup(&vips.Config{ - MaxCacheFiles: 1, - }) -} - -func imagePreview(path string, size int64) (data, error) { +func imagePreview(path string) (data, error) { preview := data{} - params := &vips.ImportParams{} - - // for big jpegs (>500kb), use shrink load - if size > 1<<19 { - params.JpegShrinkFactor.Set(jpegShrinkFactor) - } - image, err := vips.LoadImageFromFile(path, params) + image, err := vips.NewThumbnailFromFile(path, maxWidthHeight, maxWidthHeight, vips.InterestingAll) if err != nil { return preview, fmt.Errorf("%w: %w", ErrVipsLoadImage, err) } defer image.Close() preview.width, preview.height = image.Width(), image.Height() - - if err := downScale(image, imageDefault); err != nil { - return preview, fmt.Errorf("%w: %w", ErrImageDownScale, err) - } - preview.data, err = exportWebP(image) return preview, err diff --git a/pkg/preview/image_test.go b/pkg/preview/image_test.go index 0906891..fc0c92a 100644 --- a/pkg/preview/image_test.go +++ b/pkg/preview/image_test.go @@ -3,6 +3,7 @@ package preview import ( "crypto/sha256" "encoding/hex" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -23,18 +24,18 @@ func TestPreviewImage(t *testing.T) { { Name: ".jpg", SourcePath: "../../test/img/image.jpg", - DataLength: 10332, - Width: 750, - Height: 1000, - Hash: "182e0008de9bdd21a8af91ce6e6cbc57197e604886de71791133147f217c6fa0", + DataLength: 15260, + Width: 256, + Height: 341, + Hash: "916594618fa51ee0084de8525d867a53e7e9f2a9b8544d8c14eb2c17a45c7632", }, { Name: ".gif", SourcePath: "../../test/sample_minions.gif", - DataLength: 7794, - Width: 400, - Height: 200, - Hash: "846c829de9f1e4490e7025dcba99c903a7cb93289cd669b28790ffbc838e3847", + DataLength: 18700, + Width: 512, + Height: 256, + Hash: "262882df79b599b6a719f9a5f9dc0ed7ca541ca6897c290f762cd795e6edcec8", }, } @@ -43,9 +44,14 @@ func TestPreviewImage(t *testing.T) { t.Parallel() assert := assert.New(t) - preview, err := imagePreview(testCase.SourcePath, 1<<10) + preview, err := imagePreview(testCase.SourcePath) require.NoError(t, err) - assert.Len(preview.Data(), testCase.DataLength) + assert.Len( + preview.Data(), + testCase.DataLength, + fmt.Sprintf("expected = %d, actual = %d", testCase.DataLength, len(preview.Data())), + ) + width, height := preview.Resolution() assert.Equal(testCase.Width, width) assert.Equal(testCase.Height, height) diff --git a/pkg/preview/preview.go b/pkg/preview/preview.go index a496674..3a8e9f1 100644 --- a/pkg/preview/preview.go +++ b/pkg/preview/preview.go @@ -6,6 +6,8 @@ import ( "log/slog" "sync/atomic" "time" + + "github.com/davidbyttow/govips/v2/vips" ) var ( @@ -32,6 +34,29 @@ type Previewer struct { timeout time.Duration } +//nolint:gochecknoinits +func init() { + vips.LoggingSettings(func(domain string, level vips.LogLevel, msg string) { + domainSlog := slog.String("source", domain) + + switch level { + case vips.LogLevelCritical, vips.LogLevelError: + slog.Error(msg, domainSlog) + case vips.LogLevelDebug: + slog.Debug(msg, domainSlog) + case vips.LogLevelWarning: + slog.Warn(msg, domainSlog) + case vips.LogLevelMessage, vips.LogLevelInfo: + slog.Info(msg, domainSlog) + default: + slog.Info(msg, domainSlog) + } + }, vips.LogLevelError) + vips.Startup(&vips.Config{ + MaxCacheFiles: 1, + }) +} + func NewPreviewer(opts ...Option) (*Previewer, error) { preview := &Previewer{ maxImages: -1, @@ -84,7 +109,7 @@ func (p Previewer) Pull(src Source) (Data, error) { } if toImage { - preview, err := imagePreview(src.Path(), src.Size()) + preview, err := imagePreview(src.Path()) if err != nil { return defaultPreview, fmt.Errorf("%w: %w", ErrImagePreview, err) } @@ -114,3 +139,7 @@ func (p Previewer) Pull(src Source) (Data, error) { return defaultPreview, nil } + +func (p Previewer) Close() { + vips.Shutdown() +} diff --git a/pkg/preview/preview_test.go b/pkg/preview/preview_test.go index 47d3061..472935d 100644 --- a/pkg/preview/preview_test.go +++ b/pkg/preview/preview_test.go @@ -51,7 +51,7 @@ func TestPreview(t *testing.T) { }{ { source: mockSource{image: true, path: "../../test/img/image.jpg"}, - dataHash: "182e0008de9bdd21a8af91ce6e6cbc57197e604886de71791133147f217c6fa0", + dataHash: "916594618fa51ee0084de8525d867a53e7e9f2a9b8544d8c14eb2c17a45c7632", }, { source: mockSource{video: true, path: "../../test/sample.mp4"},