perf: memory usage

Author: krau

cmd/version.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	Version string = "0.7.6"
+	Version string = "0.7.7"
 )
 
 var VersionCmd = &cobra.Command{

fetcher/service.go

@@ -10,7 +10,7 @@ import (
 	"context"
 	es "errors"
 	"fmt"
-	"sync"
+	"runtime"
 
 	. "ManyACG/logger"
 
@@ -20,6 +20,7 @@ import (
 )
 
 func PostAndCreateArtwork(ctx context.Context, artwork *types.Artwork, bot *telego.Bot, storage storage.Storage, fromID int64) error {
+
 	artworkInDB, err := service.GetArtworkByURL(ctx, artwork.SourceURL)
 	if err == nil && artworkInDB != nil {
 		Logger.Debugf("Artwork %s already exists", artwork.Title)
@@ -73,22 +74,17 @@ func PostAndCreateArtwork(ctx context.Context, artwork *types.Artwork, bot *tele
 		return fmt.Errorf("error when creating artwork %s: %w", artwork.SourceURL, err)
 	}
 	go afterCreate(context.TODO(), artwork, bot, storage, fromID)
+
 	return nil
 }
 
 func afterCreate(ctx context.Context, artwork *types.Artwork, bot *telego.Bot, _ storage.Storage, fromID int64) {
-	var wg sync.WaitGroup
 	for _, picture := range artwork.Pictures {
-		wg.Add(1)
-		go func(picture *types.Picture) {
-			defer wg.Done()
-			if err := service.ProcessPictureAndUpdate(ctx, picture); err != nil {
-				Logger.Warnf("error when processing %d of artwork %s: %s", picture.Index, artwork.Title, err)
-			}
-		}(picture)
+		if err := service.ProcessPictureAndUpdate(ctx, picture); err != nil {
+			Logger.Warnf("error when processing %d of artwork %s: %s", picture.Index, artwork.Title, err)
+		}
 	}
-	wg.Wait()
-
+	runtime.GC()
 	sendNotify := fromID != 0 && bot != nil
 	artworkID, err := service.GetArtworkIDByPicture(ctx, artwork.Pictures[0])
 	artworkTitleMarkdown := common.EscapeMarkdown(artwork.Title)

service/picture.go

@@ -119,6 +119,9 @@ func ProcessPictureAndUpdate(ctx context.Context, picture *types.Picture) error
 		if err != nil {
 			return nil, err
 		}
+		defer func() {
+			fileBytes = nil
+		}()
 		hash, err := common.GetPhash(fileBytes)
 		if err != nil {
 			return nil, err

storage/webdav/webdav.go

@@ -60,6 +60,7 @@ func (w *Webdav) SavePicture(artwork *types.Artwork, picture *types.Picture) (*t
 	} else {
 		go common.PurgeFileAfter(cachePath, time.Duration(config.Cfg.Storage.CacheTTL)*time.Second)
 	}
+	fileBytes = nil
 	return storageInfo, nil
 }
 

telegram/artwork.go

@@ -7,6 +7,7 @@ import (
 	"ManyACG/storage"
 	"ManyACG/types"
 	"bytes"
+	"runtime"
 	"time"
 
 	. "ManyACG/logger"
@@ -25,29 +26,11 @@ func PostArtwork(bot *telego.Bot, artwork *types.Artwork, storage storage.Storag
 		return nil, errors.ErrNilArtwork
 	}
 
-	inputMediaPhotos := make([]telego.InputMedia, len(artwork.Pictures))
-	for i, picture := range artwork.Pictures {
-		fileBytes, err := storage.GetFile(picture.StorageInfo)
-		if err != nil {
-			Logger.Errorf("failed to get file: %s", err)
-			return nil, err
-		}
-		fileBytes, err = common.CompressImage(fileBytes, 10, 2560)
+	if len(artwork.Pictures) <= 10 {
+		inputMediaPhotos, err := getInputMediaPhotos(artwork, storage, 0, len(artwork.Pictures))
 		if err != nil {
-			Logger.Errorf("failed to compress image: %s", err)
 			return nil, err
 		}
-		photo := telegoutil.MediaPhoto(telegoutil.File(telegoutil.NameReader(bytes.NewReader(fileBytes), picture.StorageInfo.Path)))
-		if i == 0 {
-			photo = photo.WithCaption(GetArtworkHTMLCaption(artwork)).WithParseMode(telego.ModeHTML)
-		}
-		if artwork.R18 {
-			photo = photo.WithHasSpoiler()
-		}
-		inputMediaPhotos[i] = photo
-	}
-
-	if len(inputMediaPhotos) <= 10 {
 		return bot.SendMediaGroup(
 			telegoutil.MediaGroup(
 				ChannelChatID,
@@ -56,16 +39,17 @@ func PostArtwork(bot *telego.Bot, artwork *types.Artwork, storage storage.Storag
 		)
 	}
 
-	allMessages := make([]telego.Message, len(inputMediaPhotos))
-	for i := 0; i < len(inputMediaPhotos); i += 10 {
+	allMessages := make([]telego.Message, len(artwork.Pictures))
+	for i := 0; i < len(artwork.Pictures); i += 10 {
 		end := i + 10
-		if end > len(inputMediaPhotos) {
-			end = len(inputMediaPhotos)
+		if end > len(artwork.Pictures) {
+			end = len(artwork.Pictures)
 		}
-		mediaGroup := telegoutil.MediaGroup(
-			ChannelChatID,
-			inputMediaPhotos[i:end]...,
-		)
+		inputMediaPhotos, err := getInputMediaPhotos(artwork, storage, i, end)
+		if err != nil {
+			return nil, err
+		}
+		mediaGroup := telegoutil.MediaGroup(ChannelChatID, inputMediaPhotos...)
 		if i > 0 {
 			mediaGroup = mediaGroup.WithReplyParameters(&telego.ReplyParameters{
 				ChatID:    ChannelChatID,
@@ -77,7 +61,36 @@ func PostArtwork(bot *telego.Bot, artwork *types.Artwork, storage storage.Storag
 			return nil, err
 		}
 		copy(allMessages[i:], messages)
-		time.Sleep(time.Duration(int(config.Cfg.Telegram.Sleep)*len(inputMediaPhotos[i:end])) * time.Second)
+		time.Sleep(time.Duration(int(config.Cfg.Telegram.Sleep)*len(inputMediaPhotos)) * time.Second)
 	}
 	return allMessages, nil
 }
+
+// start from 0
+func getInputMediaPhotos(artwork *types.Artwork, storage storage.Storage, start, end int) ([]telego.InputMedia, error) {
+	inputMediaPhotos := make([]telego.InputMedia, end-start)
+	for i := start; i < end; i++ {
+		picture := artwork.Pictures[i]
+		fileBytes, err := storage.GetFile(picture.StorageInfo)
+		if err != nil {
+			Logger.Errorf("failed to get file: %s", err)
+			return nil, err
+		}
+		fileBytes, err = common.CompressImage(fileBytes, 10, 2560)
+		if err != nil {
+			Logger.Errorf("failed to compress image: %s", err)
+			return nil, err
+		}
+		photo := telegoutil.MediaPhoto(telegoutil.File(telegoutil.NameReader(bytes.NewReader(fileBytes), picture.StorageInfo.Path)))
+		if i == 0 {
+			photo = photo.WithCaption(GetArtworkHTMLCaption(artwork)).WithParseMode(telego.ModeHTML)
+		}
+		if artwork.R18 {
+			photo = photo.WithHasSpoiler()
+		}
+		inputMediaPhotos[i-start] = photo
+		fileBytes = nil
+	}
+	runtime.GC()
+	return inputMediaPhotos, nil
+}