wip S3 blob storage

go.mod

@@ -3,6 +3,7 @@ module github.com/ddvk/rmfakecloud
 go 1.23.3
 
 require (
+	github.com/aws/aws-sdk-go v1.49.0
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/gin-gonic/gin v1.9.1
 	github.com/golang-jwt/jwt/v4 v4.5.1
@@ -45,6 +46,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
 	github.com/gorilla/i18n v0.0.0-20150820051429-8b358169da46 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jung-kurt/gofpdf v1.16.2 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect

go.sum

@@ -46,6 +46,8 @@ github.com/adrg/xdg v0.3.0/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ
 github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
 github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY=
+github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
@@ -177,6 +179,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -582,6 +588,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app/app.go

@@ -8,12 +8,15 @@ import (
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/ddvk/rmfakecloud/internal/app/hub"
 	"github.com/ddvk/rmfakecloud/internal/config"
 	"github.com/ddvk/rmfakecloud/internal/hwr"
 	"github.com/ddvk/rmfakecloud/internal/storage"
 
 	"github.com/ddvk/rmfakecloud/internal/storage/fs"
+	"github.com/ddvk/rmfakecloud/internal/storage/s3"
 	"github.com/ddvk/rmfakecloud/internal/ui"
 
 	"github.com/gin-gonic/gin"
@@ -100,11 +103,21 @@ func NewApp(cfg *config.Config) App {
 
 	fsStorage := fs.NewStorage(cfg)
 	usrs, err := fsStorage.GetUsers()
-
 	if err != nil {
 		log.Warn(err)
 	}
 
+	s3_path_style := true
+	blobStorage, err := s3.NewS3BlobStorage(&aws.Config{
+		Region:           aws.String(cfg.AWSRegion),
+		Credentials:      credentials.NewStaticCredentials(cfg.AWSAccessKey, cfg.AWSSecretKey, ""),
+		Endpoint:         aws.String(cfg.AWSEndpoint),
+		S3ForcePathStyle: aws.Bool(s3_path_style),
+	}, cfg.S3BucketName)
+	if err != nil {
+		log.Error(err)
+	}
+
 	if len(usrs) == 0 {
 		log.Warn("No users found, the first login will create a user")
 		//TODO: not thread safe
@@ -138,7 +151,7 @@ func NewApp(cfg *config.Config) App {
 		docStorer:     fsStorage,
 		userStorer:    fsStorage,
 		metaStorer:    fsStorage,
-		blobStorer:    fsStorage,
+		blobStorer:    blobStorage,
 		hub:           ntfHub,
 		codeConnector: codeConnector,
 		hwrClient: &hwr.HWRClient{
@@ -150,7 +163,7 @@ func NewApp(cfg *config.Config) App {
 	uiApp := ui.New(cfg, fsStorage, codeConnector, ntfHub, fsStorage, fsStorage)
 	uiApp.RegisterRoutes(router)
 
-	storageapp := fs.NewApp(cfg, fsStorage)
+	storageapp := storage.NewApp(cfg, fsStorage, fsStorage, blobStorage)
 	storageapp.RegisterRoutes(router)
 
 	return app

internal/app/handlers.go

@@ -22,7 +22,6 @@ import (
 	"github.com/ddvk/rmfakecloud/internal/integrations"
 	"github.com/ddvk/rmfakecloud/internal/messages"
 	"github.com/ddvk/rmfakecloud/internal/storage"
-	"github.com/ddvk/rmfakecloud/internal/storage/fs"
 	"github.com/gin-gonic/gin"
 	"github.com/golang-jwt/jwt/v4"
 	"github.com/gorilla/websocket"
@@ -745,7 +744,7 @@ func (app *App) syncUpdateRootV3(c *gin.Context) {
 	}
 
 	uid := userID(c)
-	newgeneration, err := app.blobStorer.StoreBlob(uid, RootHash, bytes.NewBufferString(rootv3.Hash), rootv3.Generation)
+	newgeneration, err := app.userStorer.UpdateRoot(uid, bytes.NewBufferString(rootv3.Hash), rootv3.Generation)
 	if err != nil {
 		log.Error(err)
 		c.AbortWithStatus(http.StatusInternalServerError)
@@ -791,8 +790,8 @@ func crcJSON(c *gin.Context, status int, msg any) {
 
 func (app *App) syncGetRootV3(c *gin.Context) {
 	uid := userID(c)
-	reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash)
-	if err == fs.ErrorNotFound {
+	roothash, generation, err := app.userStorer.GetRoot(uid)
+	if err == storage.ErrorNotFound {
 		log.Warn("No root file found, assuming this is a new account")
 		c.JSON(http.StatusNotFound, gin.H{"message": "root not found"})
 		return
@@ -804,27 +803,18 @@ func (app *App) syncGetRootV3(c *gin.Context) {
 		return
 	}
 
-	roothash, err := io.ReadAll(reader)
-	if err != nil {
-		log.Error(err)
-		c.AbortWithStatus(http.StatusInternalServerError)
-		return
-	}
-
 	c.JSON(http.StatusOK, messages.SyncRootV3Response{
 		Generation: generation,
-		Hash:       string(roothash),
+		Hash:       roothash,
 	})
 }
 
 func (app *App) syncGetRootV4(c *gin.Context) {
 	uid := userID(c)
-	reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash)
-	if err == fs.ErrorNotFound {
+	roothash, generation, err := app.userStorer.GetRoot(uid)
+	if err == storage.ErrorNotFound {
 		log.Warn("No root file found, assuming this is a new account")
-		crcJSON(c, http.StatusOK, messages.SyncRootV4Response{
-			SchemaVersion: SchemaVersion,
-		})
+		c.JSON(http.StatusNotFound, gin.H{"message": "root not found"})
 		return
 	}
 
@@ -834,12 +824,6 @@ func (app *App) syncGetRootV4(c *gin.Context) {
 		return
 	}
 
-	roothash, err := io.ReadAll(reader)
-	if err != nil {
-		log.Error(err)
-		c.AbortWithStatus(http.StatusInternalServerError)
-		return
-	}
 	crcJSON(c, http.StatusOK, messages.SyncRootV4Response{
 		Generation:    generation,
 		Hash:          string(roothash),
@@ -880,8 +864,8 @@ func (app *App) blobStorageRead(c *gin.Context) {
 	uid := userID(c)
 	blobID := common.ParamS(fileKey, c)
 
-	reader, _, size, crc32c, err := app.blobStorer.LoadBlob(uid, blobID)
-	if err == fs.ErrorNotFound {
+	reader, size, crc32c, err := app.blobStorer.LoadBlob(uid, blobID)
+	if err == storage.ErrorNotFound {
 		log.Warn(err)
 		c.AbortWithStatus(http.StatusNotFound)
 		return
@@ -905,7 +889,7 @@ func (app *App) blobStorageWrite(c *gin.Context) {
 	hash := c.GetHeader(common.CRC32CHashHeader)
 	log.Debugf("TODO: check/save etc. write file '%s', hash '%s'", fileName, hash)
 
-	_, err := app.blobStorer.StoreBlob(uid, blobID, c.Request.Body, 0)
+	err := app.blobStorer.StoreBlob(uid, blobID, c.Request.Body)
 	if err != nil {
 		log.Error(err)
 		c.AbortWithStatus(http.StatusInternalServerError)

internal/config/config.go

@@ -48,6 +48,14 @@ const (
 	envJWTSecretKey     = "JWT_SECRET_KEY"
 	envRegistrationOpen = "OPEN_REGISTRATION"
 
+	// s3
+	envAWSAccessKey = "AWS_ACCESS_KEY_ID"
+	envAWSSecretKey = "AWS_SECRET_ACCESS_KEY"
+	envAWSRegion    = "AWS_REGION"
+	envAWSEndpoint  = "AWS_ENDPOINT_URL"
+	envS3PathStyle  = "S3_PATH_STYLE"
+	envS3BucketName = "S3_BUCKET_NAME"
+
 	// envSMTPServer the mail server
 	envSMTPServer = "RM_SMTP_SERVER"
 	// envSMTPUsername the username for the mail server
@@ -77,8 +85,8 @@ const (
 
 // Config config
 type Config struct {
-	Port              string
-	StorageURL        string
+	Port       string
+	StorageURL string
 	//only https
 	CloudHost         string
 	DataDir           string
@@ -93,6 +101,13 @@ type Config struct {
 	HWRHmac           string
 	HTTPSCookie       bool
 	TrustProxy        bool
+
+	AWSAccessKey string
+	AWSSecretKey string
+	AWSRegion    string
+	AWSEndpoint  string
+	S3PathStyle  bool
+	S3BucketName string
 }
 
 // Verify verify
@@ -107,6 +122,12 @@ func (cfg *Config) Verify() {
 		log.Warnln(envHTTPSCookie + " is not set, use only when not using https!")
 	}
 
+	if cfg.AWSEndpoint != "" {
+		if _, err := url.Parse(cfg.AWSEndpoint); err != nil {
+			log.Fatal(envAWSEndpoint + " is not a valid URL: " + err.Error())
+		}
+	}
+
 	if cfg.SMTPConfig == nil {
 		log.Warnln("smtp not configured, no emails will be sent")
 	}
@@ -170,7 +191,7 @@ func FromEnv() *Config {
 		uploadURL = "https://" + DefaultHost
 	} else {
 		u, err := url.Parse(uploadURL)
-		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == ""  {
+		if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
 			log.Fatalf("%s '%s' cannot be parsed, or missing scheme (http|https) %v", EnvStorageURL, uploadURL, err)
 		}
 		if u.Port() != "" {
@@ -211,11 +232,12 @@ func FromEnv() *Config {
 	}
 
 	trustProxy, _ := strconv.ParseBool(os.Getenv(envTrustProxy))
+	s3PathStyle, _ := strconv.ParseBool(os.Getenv(envS3PathStyle))
 
 	cfg := Config{
 		Port:              port,
 		StorageURL:        uploadURL,
-		CloudHost: cloudHost,
+		CloudHost:         cloudHost,
 		DataDir:           dataDir,
 		JWTSecretKey:      dk,
 		JWTRandom:         jwtGenerated,
@@ -226,7 +248,19 @@ func FromEnv() *Config {
 		HWRHmac:           os.Getenv(envHwrHmac),
 		HTTPSCookie:       httpsCookie,
 		TrustProxy:        trustProxy,
+
+		AWSAccessKey: os.Getenv(envAWSAccessKey),
+		AWSSecretKey: os.Getenv(envAWSSecretKey),
+		AWSRegion:    os.Getenv(envAWSRegion),
+		AWSEndpoint:  os.Getenv(envAWSEndpoint),
+		S3PathStyle:  s3PathStyle,
+		S3BucketName: os.Getenv(envS3BucketName),
 	}
+
+	if cfg.AWSEndpoint != "" && cfg.AWSRegion == "" {
+		cfg.AWSRegion = "us-west-1"
+	}
+
 	return &cfg
 }
 
@@ -251,6 +285,14 @@ General:
 	%s Send auth cookie only via https
 	%s	Trust the proxy for X-Forwarded-For/X-Real-IP (set only if behind a proxy)
 
+S3-compatible storage:
+	%s	Your AWS access key
+	%s	Your AWS secret key
+	%s	Your AWS region
+	%s	If not using official AWS, endpoint to use instead
+	%s	Force path style for non-domain wide buckets (eg. minio, ...)
+	%s	The S3 bucket to use
+
 Emails, smtp:
 	%s
 	%s
@@ -279,6 +321,13 @@ myScript hwr (needs a developer account):
 		envHTTPSCookie,
 		envTrustProxy,
 
+		envAWSAccessKey,
+		envAWSSecretKey,
+		envAWSRegion,
+		envAWSEndpoint,
+		envS3PathStyle,
+		envS3BucketName,
+
 		envSMTPServer,
 		envSMTPUsername,
 		envSMTPPassword,