diff --git a/cmd/scripts/prose-imgs-migrate/main.go b/cmd/scripts/prose-imgs-migrate/main.go index e8781e1b..f8c6f481 100644 --- a/cmd/scripts/prose-imgs-migrate/main.go +++ b/cmd/scripts/prose-imgs-migrate/main.go @@ -64,7 +64,7 @@ func images(logger *slog.Logger, dbh db.DB, st storage.StorageServe, bucket sst. rdr, _, err := st.GetObject(imgBucket, posts.Filename) if err != nil { logger.Error("get object", "err", err) - return err + continue } err = upload(logger, st, bucket, posts.Filename, rdr) if err != nil { @@ -86,6 +86,9 @@ func main() { bail(err) for _, user := range users { + if user.Name != "erock" { + continue + } logger.Info("migrating user images", "user", user.Name) bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID)) diff --git a/db/db.go b/db/db.go index 2671f69c..45263c3f 100644 --- a/db/db.go +++ b/db/db.go @@ -403,6 +403,7 @@ type DB interface { InsertFeedItems(postID string, items []*FeedItem) error FindFeedItemsByPostID(postID string) ([]*FeedItem, error) + UpsertProject(userID, name, projectDir string) (*Project, error) InsertProject(userID, name, projectDir string) (string, error) UpdateProject(userID, name string) error UpdateProjectAcl(userID, name string, acl ProjectAcl) error diff --git a/db/postgres/storage.go b/db/postgres/storage.go index 809f64a2..28d43d06 100644 --- a/db/postgres/storage.go +++ b/db/postgres/storage.go @@ -1990,3 +1990,23 @@ func (me *PsqlDB) AddPicoPlusUser(username, email, paymentType, txId string) err return tx.Commit() } + +func (me *PsqlDB) UpsertProject(userID, projectName, projectDir string) (*db.Project, error) { + project, err := me.FindProjectByName(userID, projectName) + if err == nil { + // this just updates the `createdAt` timestamp, useful for book-keeping + err = me.UpdateProject(userID, projectName) + if err != nil { + me.Logger.Error("could not update project", "err", err) + return nil, err + } + return project, nil + } + + _, err = me.InsertProject(userID, projectName, projectName) + if err != nil { + me.Logger.Error("could not create project", "err", err) + return nil, err + } + return me.FindProjectByName(userID, projectName) +} diff --git a/db/stub/stub.go b/db/stub/stub.go index 1308f4fe..31707d7c 100644 --- a/db/stub/stub.go +++ b/db/stub/stub.go @@ -213,6 +213,10 @@ func (me *StubDB) FindFeedItemsByPostID(postID string) ([]*db.FeedItem, error) { return []*db.FeedItem{}, notImpl } +func (me *StubDB) UpsertProject(userID, name, projectDir string) (*db.Project, error) { + return nil, notImpl +} + func (me *StubDB) InsertProject(userID, name, projectDir string) (string, error) { return "", notImpl } diff --git a/filehandlers/imgs/handler.go b/filehandlers/imgs/handler.go index 53240094..9faf6882 100644 --- a/filehandlers/imgs/handler.go +++ b/filehandlers/imgs/handler.go @@ -1,15 +1,15 @@ package uploadimgs import ( + "bytes" "encoding/binary" "fmt" "io" "net/http" "os" "path/filepath" - "time" - "slices" + "strings" "github.com/charmbracelet/ssh" exifremove "github.com/neurosnap/go-exif-remove" @@ -17,6 +17,7 @@ import ( "github.com/picosh/pico/shared" "github.com/picosh/pico/shared/storage" "github.com/picosh/pobj" + sst "github.com/picosh/pobj/storage" sendutils "github.com/picosh/send/utils" "github.com/picosh/utils" ) @@ -24,13 +25,13 @@ import ( var Space = "imgs" type PostMetaData struct { - *db.Post - OrigText []byte - Cur *db.Post - Tags []string - User *db.User - *sendutils.FileEntry - FeatureFlag *db.FeatureFlag + Text []byte + FileSize int + TotalFileSize int + Filename string + User *db.User + FeatureFlag *db.FeatureFlag + Bucket sst.Bucket } type UploadImgHandler struct { @@ -51,28 +52,61 @@ func (h *UploadImgHandler) getObjectPath(fpath string) string { return filepath.Join("prose", fpath) } -func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) { +func (h *UploadImgHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { + var fileList []os.FileInfo + user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"]) if err != nil { - return nil, nil, err + return fileList, err } - cleanFilename := filepath.Base(entry.Filepath) + cleanFilename := fpath + + bucketName := shared.GetAssetBucketName(user.ID) + bucket, err := h.Storage.GetBucket(bucketName) + if err != nil { + return fileList, err + } if cleanFilename == "" || cleanFilename == "." { - return nil, nil, os.ErrNotExist + name := cleanFilename + if name == "" { + name = "/" + } + + info := &sendutils.VirtualFile{ + FName: name, + FIsDir: true, + } + + fileList = append(fileList, info) + } else { + fp := h.getObjectPath(cleanFilename) + if fp != "/" && isDir { + fp += "/" + } + + foundList, err := h.Storage.ListObjects(bucket, fp, recursive) + if err != nil { + return fileList, err + } + + fileList = append(fileList, foundList...) } - post, err := h.DBPool.FindPostWithFilename(cleanFilename, user.ID, Space) + return fileList, nil +} + +func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) { + user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"]) if err != nil { return nil, nil, err } - fileInfo := &sendutils.VirtualFile{ - FName: post.Filename, - FIsDir: false, - FSize: int64(post.FileSize), - FModTime: *post.UpdatedAt, + cleanFilename := filepath.Base(entry.Filepath) + + if cleanFilename == "" || cleanFilename == "." { + return nil, nil, os.ErrNotExist } bucket, err := h.Storage.GetBucket(shared.GetAssetBucketName(user.ID)) @@ -80,13 +114,19 @@ func (h *UploadImgHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.F return nil, nil, err } - contents, _, err := h.Storage.GetObject(bucket, h.getObjectPath(post.Filename)) + contents, info, err := h.Storage.GetObject(bucket, h.getObjectPath(cleanFilename)) if err != nil { return nil, nil, err } - reader := pobj.NewAllReaderAt(contents) + fileInfo := &sendutils.VirtualFile{ + FName: cleanFilename, + FIsDir: false, + FSize: info.Size, + FModTime: info.LastModified, + } + return fileInfo, reader, nil } @@ -125,53 +165,33 @@ func (h *UploadImgHandler) Write(s ssh.Session, entry *sendutils.FileEntry) (str } } - now := time.Now() fileSize := binary.Size(text) - shasum := utils.Shasum(text) - slug := utils.SanitizeFileExt(filename) - - nextPost := db.Post{ - Filename: filename, - Slug: slug, - PublishAt: &now, - Text: string(text), - MimeType: mimeType, - FileSize: fileSize, - Shasum: shasum, - } - - post, err := h.DBPool.FindPostWithFilename( - nextPost.Filename, - user.ID, - Space, - ) - if err != nil { - logger.Info("unable to find image, continuing", "filename", nextPost.Filename, "err", err.Error()) - } - featureFlag := shared.FindPlusFF(h.DBPool, h.Cfg, user.ID) - metadata := PostMetaData{ - OrigText: text, - Post: &nextPost, - User: user, - FileEntry: entry, - Cur: post, - FeatureFlag: featureFlag, - } - if post != nil { - metadata.Post.PublishAt = post.PublishAt + bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID)) + if err != nil { + return "", err } - err = h.writeImg(s, &metadata) + totalFileSize, err := h.Storage.GetBucketQuota(bucket) if err != nil { - logger.Error("could not write img", "err", err.Error()) + logger.Error("bucket quota", "err", err) return "", err } - totalFileSize, err := h.DBPool.FindTotalSizeForUser(user.ID) + metadata := PostMetaData{ + Filename: filename, + FileSize: fileSize, + Text: text, + User: user, + FeatureFlag: featureFlag, + Bucket: bucket, + TotalFileSize: int(totalFileSize), + } + + err = h.writeImg(&metadata) if err != nil { - logger.Error("could not find total storage size for user", "err", err.Error()) + logger.Error("could not write img", "err", err.Error()) return "", err } @@ -185,7 +205,7 @@ func (h *UploadImgHandler) Write(s ssh.Session, entry *sendutils.FileEntry) (str str := fmt.Sprintf( "%s (space: %.2f/%.2fGB, %.2f%%)", url, - utils.BytesToGB(totalFileSize), + utils.BytesToGB(metadata.TotalFileSize+fileSize), utils.BytesToGB(maxSize), (float32(totalFileSize)/float32(maxSize))*100, ) @@ -206,30 +226,93 @@ func (h *UploadImgHandler) Delete(s ssh.Session, entry *sendutils.FileEntry) err "filename", filename, ) - post, err := h.DBPool.FindPostWithFilename( - filename, - user.ID, - Space, - ) + bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID)) if err != nil { - logger.Info("unable to find image, continuing", "err", err.Error()) return err } - err = h.DBPool.RemovePosts([]string{post.ID}) + logger.Info("deleting image") + err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename)) if err != nil { - logger.Error("error removing image", "error", err) - return fmt.Errorf("error for %s: %v", filename, err) + return err } - bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(user.ID)) + return nil +} + +func (h *UploadImgHandler) validateImg(data *PostMetaData) (bool, error) { + fileMax := data.FeatureFlag.Data.FileMax + if int64(data.FileSize) > fileMax { + return false, fmt.Errorf("ERROR: file (%s) has exceeded maximum file size (%d bytes)", data.Filename, fileMax) + } + + storageMax := data.FeatureFlag.Data.StorageMax + if uint64(data.TotalFileSize+data.FileSize) > storageMax { + return false, fmt.Errorf("ERROR: user (%s) has exceeded (%d bytes) max (%d bytes)", data.User.Name, data.TotalFileSize, storageMax) + } + + if !utils.IsExtAllowed(data.Filename, h.Cfg.AllowedExt) { + extStr := strings.Join(h.Cfg.AllowedExt, ",") + err := fmt.Errorf( + "ERROR: (%s) invalid file, format must be (%s), skipping", + data.Filename, + extStr, + ) + return false, err + } + + return true, nil +} + +func (h *UploadImgHandler) metaImg(data *PostMetaData) error { + // if the file is empty that means we should delete it + // so we can skip all the meta info + if data.FileSize == 0 { + return nil + } + + // make sure we have a bucket + bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID)) if err != nil { return err } - logger.Info("deleting image") - err = h.Storage.DeleteObject(bucket, h.getObjectPath(filename)) + // make sure we have a prose project to upload to + _, err = h.DBPool.UpsertProject(data.User.ID, "prose", "prose") + if err != nil { + return err + } + + reader := bytes.NewReader([]byte(data.Text)) + _, _, err = h.Storage.PutObject( + bucket, + h.getObjectPath(data.Filename), + sendutils.NopReaderAtCloser(reader), + &sendutils.FileEntry{}, + ) + if err != nil { + return err + } + + return nil +} + +func (h *UploadImgHandler) writeImg(data *PostMetaData) error { + valid, err := h.validateImg(data) + if !valid { + return err + } + + logger := h.Cfg.Logger + logger = shared.LoggerWithUser(logger, data.User) + logger = logger.With( + "filename", data.Filename, + ) + + logger.Info("uploading image") + err = h.metaImg(data) if err != nil { + logger.Error("meta img", "err", err) return err } diff --git a/filehandlers/imgs/img.go b/filehandlers/imgs/img.go deleted file mode 100644 index 83b296a6..00000000 --- a/filehandlers/imgs/img.go +++ /dev/null @@ -1,161 +0,0 @@ -package uploadimgs - -import ( - "bytes" - "fmt" - "strings" - "time" - - "github.com/charmbracelet/ssh" - "github.com/picosh/pico/db" - "github.com/picosh/pico/shared" - sendutils "github.com/picosh/send/utils" - "github.com/picosh/utils" -) - -func (h *UploadImgHandler) validateImg(data *PostMetaData) (bool, error) { - totalFileSize, err := h.DBPool.FindTotalSizeForUser(data.User.ID) - if err != nil { - return false, err - } - - fileMax := data.FeatureFlag.Data.FileMax - if int64(data.FileSize) > fileMax { - return false, fmt.Errorf("ERROR: file (%s) has exceeded maximum file size (%d bytes)", data.Filename, fileMax) - } - - storageMax := data.FeatureFlag.Data.StorageMax - if uint64(totalFileSize+data.FileSize) > storageMax { - return false, fmt.Errorf("ERROR: user (%s) has exceeded (%d bytes) max (%d bytes)", data.User.Name, totalFileSize, storageMax) - } - - if !utils.IsExtAllowed(data.Filepath, h.Cfg.AllowedExt) { - extStr := strings.Join(h.Cfg.AllowedExt, ",") - err := fmt.Errorf( - "ERROR: (%s) invalid file, format must be (%s), skipping", - data.Filename, - extStr, - ) - return false, err - } - - return true, nil -} - -func (h *UploadImgHandler) metaImg(data *PostMetaData) error { - // if the file is empty that means we should delete it - // so we can skip all the meta info - if data.FileSize == 0 { - return nil - } - - bucket, err := h.Storage.UpsertBucket(shared.GetAssetBucketName(data.User.ID)) - if err != nil { - return err - } - - reader := bytes.NewReader([]byte(data.Text)) - - fname, _, err := h.Storage.PutObject( - bucket, - h.getObjectPath(data.Filename), - sendutils.NopReaderAtCloser(reader), - &sendutils.FileEntry{}, - ) - if err != nil { - return err - } - - data.Data = db.PostData{ - ImgPath: fname, - } - - data.Text = "" - - return nil -} - -func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error { - valid, err := h.validateImg(data) - if !valid { - return err - } - user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"]) - if err != nil { - return err - } - - logger := h.Cfg.Logger - logger = shared.LoggerWithUser(logger, user) - - err = h.metaImg(data) - if err != nil { - logger.Error("could not get meta for img", "err", err.Error()) - return err - } - - modTime := time.Now() - - if data.Mtime > 0 { - modTime = time.Unix(data.Mtime, 0) - } - - logger = logger.With( - "filename", data.Filename, - ) - - if data.Cur == nil { - logger.Info("file not found, adding record") - insertPost := db.Post{ - UserID: user.ID, - Space: Space, - - Data: data.Data, - Description: data.Description, - Filename: data.Filename, - FileSize: data.FileSize, - Hidden: data.Hidden, - MimeType: data.MimeType, - PublishAt: data.PublishAt, - Shasum: data.Shasum, - Slug: data.Slug, - Text: data.Text, - Title: data.Title, - UpdatedAt: &modTime, - } - _, err := h.DBPool.InsertPost(&insertPost) - if err != nil { - logger.Error("post could not create", "err", err.Error()) - return fmt.Errorf("error for %s: %v", data.Filename, err) - } - } else { - if data.Shasum == data.Cur.Shasum && modTime.Equal(*data.Cur.UpdatedAt) { - logger.Info("image found, but image is identical, skipping") - return nil - } - - logger.Info("file found, updating record") - - updatePost := db.Post{ - ID: data.Cur.ID, - - Data: data.Data, - FileSize: data.FileSize, - Description: data.Description, - PublishAt: data.PublishAt, - Slug: data.Slug, - Shasum: data.Shasum, - Text: data.Text, - Title: data.Title, - Hidden: data.Hidden, - UpdatedAt: &modTime, - } - _, err = h.DBPool.UpdatePost(&updatePost) - if err != nil { - logger.Error("post could not update", "err", err.Error()) - return fmt.Errorf("error for %s: %v", data.Filename, err) - } - } - - return nil -} diff --git a/filehandlers/post_handler.go b/filehandlers/post_handler.go index 72492bbd..ffdcac84 100644 --- a/filehandlers/post_handler.go +++ b/filehandlers/post_handler.go @@ -45,6 +45,10 @@ func NewScpPostHandler(dbpool db.DB, cfg *shared.ConfigSite, hooks ScpFileHooks) } } +func (r *ScpUploadHandler) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { + return BaseList(s, fpath, isDir, recursive, []string{r.Cfg.Space}, r.DBPool) +} + func (h *ScpUploadHandler) Read(s ssh.Session, entry *sendutils.FileEntry) (os.FileInfo, sendutils.ReaderAtCloser, error) { user, err := h.DBPool.FindUser(s.Permissions().Extensions["user_id"]) if err != nil { diff --git a/filehandlers/router_handler.go b/filehandlers/router_handler.go index 241500b0..403367cd 100644 --- a/filehandlers/router_handler.go +++ b/filehandlers/router_handler.go @@ -15,6 +15,7 @@ import ( ) type ReadWriteHandler interface { + List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) Write(ssh.Session, *utils.FileEntry) (string, error) Read(ssh.Session, *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) Delete(ssh.Session, *utils.FileEntry) error @@ -39,8 +40,8 @@ func NewFileHandlerRouter(cfg *shared.ConfigSite, dbpool db.DB, mapper map[strin } } -func (r *FileHandlerRouter) findHandler(entry *utils.FileEntry) (ReadWriteHandler, error) { - fext := filepath.Ext(entry.Filepath) +func (r *FileHandlerRouter) findHandler(fp string) (ReadWriteHandler, error) { + fext := filepath.Ext(fp) handler, ok := r.FileMap[fext] if !ok { hand, hasFallback := r.FileMap["fallback"] @@ -57,7 +58,7 @@ func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string return "", os.ErrInvalid } - handler, err := r.findHandler(entry) + handler, err := r.findHandler(entry.Filepath) if err != nil { return "", err } @@ -65,7 +66,7 @@ func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string } func (r *FileHandlerRouter) Delete(s ssh.Session, entry *utils.FileEntry) error { - handler, err := r.findHandler(entry) + handler, err := r.findHandler(entry.Filepath) if err != nil { return err } @@ -73,13 +74,49 @@ func (r *FileHandlerRouter) Delete(s ssh.Session, entry *utils.FileEntry) error } func (r *FileHandlerRouter) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { - handler, err := r.findHandler(entry) + handler, err := r.findHandler(entry.Filepath) if err != nil { return nil, nil, err } return handler.Read(s, entry) } +func (r *FileHandlerRouter) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { + files := []os.FileInfo{} + for key, handler := range r.FileMap { + // TODO: hack because we have duplicate keys for .md and .css + if key == ".css" { + continue + } + + ff, err := handler.List(s, fpath, isDir, recursive) + if err != nil { + r.GetLogger().Error("handler list", "err", err) + continue + } + files = append(files, ff...) + } + return files, nil +} + +func (r *FileHandlerRouter) GetLogger() *slog.Logger { + return r.Cfg.Logger +} + +func (r *FileHandlerRouter) Validate(s ssh.Session) error { + user, err := r.DBPool.FindUser(s.Permissions().Extensions["user_id"]) + if err != nil { + return err + } + + r.Cfg.Logger.Info( + "attempting to upload files", + "user", user.Name, + "space", r.Cfg.Space, + ) + return nil +} + func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces []string, dbpool db.DB) ([]os.FileInfo, error) { var fileList []os.FileInfo user, err := dbpool.FindUser(s.Permissions().Extensions["user_id"]) @@ -112,7 +149,6 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces [] } } else { for _, space := range spaces { - p, e := dbpool.FindPostWithFilename(cleanFilename, user.ID, space) if e != nil { err = e @@ -143,25 +179,3 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces [] return fileList, nil } - -func (r *FileHandlerRouter) List(s ssh.Session, fpath string, isDir bool, recursive bool) ([]os.FileInfo, error) { - return BaseList(s, fpath, isDir, recursive, r.Spaces, r.DBPool) -} - -func (r *FileHandlerRouter) GetLogger() *slog.Logger { - return r.Cfg.Logger -} - -func (r *FileHandlerRouter) Validate(s ssh.Session) error { - user, err := r.DBPool.FindUser(s.Permissions().Extensions["user_id"]) - if err != nil { - return err - } - - r.Cfg.Logger.Info( - "attempting to upload files", - "user", user.Name, - "space", r.Cfg.Space, - ) - return nil -} diff --git a/pgs/uploader.go b/pgs/uploader.go index efec2bfc..014bda7d 100644 --- a/pgs/uploader.go +++ b/pgs/uploader.go @@ -279,24 +279,10 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *sendutils.FileEntry) (s // find, create, or update project if we haven't already done it if project == nil { - project, err = h.DBPool.FindProjectByName(user.ID, projectName) - if err == nil { - err = h.DBPool.UpdateProject(user.ID, projectName) - if err != nil { - logger.Error("could not update project", "err", err.Error()) - return "", err - } - } else { - _, err = h.DBPool.InsertProject(user.ID, projectName, projectName) - if err != nil { - logger.Error("could not create project", "err", err.Error()) - return "", err - } - project, err = h.DBPool.FindProjectByName(user.ID, projectName) - if err != nil { - logger.Error("could not find project", "err", err.Error()) - return "", err - } + project, err = h.DBPool.UpsertProject(user.ID, projectName, projectName) + if err != nil { + logger.Error("upsert project", "err", err.Error()) + return "", err } setProject(s, project) } diff --git a/pgs/wish.go b/pgs/wish.go index da0e5fcd..a6f3f458 100644 --- a/pgs/wish.go +++ b/pgs/wish.go @@ -148,10 +148,6 @@ func WishMiddleware(handler *UploadAssetHandler) wish.Middleware { err := opts.fzf(projectName) opts.bail(err) return - } else if cmd == "stats" { - err := opts.statsByProject(projectName) - opts.bail(err) - return } else if cmd == "link" { linkCmd, write := flagSet("link", sesh) linkTo := linkCmd.String("to", "", "symbolic link to this project") diff --git a/prose/ssh.go b/prose/ssh.go index 547bffad..a67c0729 100644 --- a/prose/ssh.go +++ b/prose/ssh.go @@ -85,7 +85,6 @@ func StartSshServer() { "fallback": uploadimgs.NewUploadImgHandler(dbh, cfg, st), } handler := filehandlers.NewFileHandlerRouter(cfg, dbh, fileMap) - handler.Spaces = []string{cfg.Space, "imgs"} sshAuth := shared.NewSshAuthHandler(dbh, logger, cfg) s, err := wish.NewServer(