Merge pull request #131 from covalenthq/develop

Merge Multipart file handling and security updates

.github/dependabot.yml

@@ -12,5 +12,5 @@ updates:
     target-branch: "develop"
     reviewers:
       - "noslav"
-      - "sudeepdino008"
+      - "rogarcia"
 

.github/workflows/docker-ci.yml

@@ -3,16 +3,20 @@ name: docker-ci
 on:
   push:
     branches:
-    - "main"
+      - "main"
   pull_request:
-    branches: 
-    - "main"
-    - "develop"
+    branches:
+      - "main"
+      - "develop"
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.6.0
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Artifact Registry/GCR
         uses: docker/login-action@v2
         with:

core/support.go

@@ -17,7 +17,7 @@ const (
 	// IpfsPinnerVersionMinor is Minor version component of the current release
 	IpfsPinnerVersionMinor = 2
 	// IpfsPinnerVersionPatch is Patch version component of the current release
-	IpfsPinnerVersionPatch = 1
+	IpfsPinnerVersionPatch = 3
 	clientIdentifier       = "ipfs-pinner" // Client identifier to advertise over the network
 )
 

go.mod

@@ -13,13 +13,13 @@ require (
 	github.com/ipfs/kubo v0.29.0
 	github.com/ipld/go-car v0.6.2
 	github.com/ipld/go-ipld-prime v0.21.0
-	github.com/multiformats/go-multiaddr v0.12.4
+	github.com/multiformats/go-multiaddr v0.14.0
 	github.com/multiformats/go-multibase v0.2.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/pkg/errors v0.9.1
 	github.com/web3-storage/go-ucanto v0.1.0
 	github.com/ybbus/httpretry v1.0.2
-	golang.org/x/oauth2 v0.21.0
+	golang.org/x/oauth2 v0.24.0
 )
 
 require (

go.sum

@@ -519,8 +519,8 @@ github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a
 github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
 github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
 github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
-github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc=
-github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
+github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU=
+github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4=
 github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
@@ -941,8 +941,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
-golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
+golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

server/main.go

@@ -57,6 +57,8 @@ var (
 
 	DOWNLOAD_TIMEOUT = 12 * time.Minute // download can take a lot of time if it's not locally present
 	UPLOAD_TIMEOUT   = 60 * time.Second // uploads of around 6MB files happen in less than 10s typically
+
+	maxMemory = int64(100 << 20) // 100 MB
 )
 
 func main() {
@@ -166,15 +168,18 @@ func uploadHttpHandler(node pinner.PinnerNode) http.Handler {
 			respondError(w, err)
 			return
 		}
-		ccid, err := uploadHandler(contents, node)
-		if err != nil {
-			respondError(w, err)
-			return
-		} else {
-			succ_str := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String())
-			_, err := w.Write([]byte(succ_str))
+
+		for _, v := range contents {
+			ccid, err := uploadHandler(v, node)
 			if err != nil {
-				log.Println("error writing data to connection: %w", err)
+				respondError(w, err)
+				return
+			} else {
+				succStr := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String())
+				_, err := w.Write([]byte(succStr))
+				if err != nil {
+					log.Println("error writing data to connection: %w", err)
+				}
 			}
 		}
 	}
@@ -209,45 +214,47 @@ func cidHttpHandler(node pinner.PinnerNode) http.Handler {
 			return
 		}
 
-		ccid, err := cidHandler(contents, node)
-		if err != nil {
-			respondError(w, err)
-			return
-		} else {
-			succ_str := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String())
-			_, err := w.Write([]byte(succ_str))
+		for _, v := range contents {
+			ccid, err := cidHandler(v, node)
 			if err != nil {
-				log.Println("error writing data to connection: %w", err)
+				respondError(w, err)
+				return
+			} else {
+				succStr := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String())
+				_, err := w.Write([]byte(succStr))
+				if err != nil {
+					log.Println("error writing data to connection: %w", err)
+				}
 			}
 		}
 	}
 
 	return http.HandlerFunc(fn)
 }
 
-func readContentFromRequest(r *http.Request) (string, error) {
-	mreader, err := r.MultipartReader()
+func readContentFromRequest(r *http.Request) (map[string][]byte, error) {
+	err := r.ParseMultipartForm(maxMemory)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
-	var contents string = ""
-
-	for {
-		part, err := mreader.NextPart()
-		if err == io.EOF {
-			break
-		}
+	files := make(map[string][]byte)
+	for _, fileHeaders := range r.MultipartForm.File {
+		for _, fileHeader := range fileHeaders {
+			file, err := fileHeader.Open()
+			if err != nil {
+				return nil, err
+			}
+			defer file.Close()
 
-		pcontents, err := io.ReadAll(part)
-		if err != nil {
-			return "", err
+			data, err := io.ReadAll(file)
+			if err != nil {
+				return nil, err
+			}
+			files[fileHeader.Filename] = data
 		}
-
-		contents += string(pcontents)
 	}
-
-	return contents, nil
+	return files, nil
 }
 
 func recoveryWrapper(h http.Handler) http.Handler {
@@ -271,11 +278,11 @@ func recoveryWrapper(h http.Handler) http.Handler {
 	})
 }
 
-func cidHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) {
+func cidHandler(contents []byte, node pinner.PinnerNode) (cid.Cid, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), UPLOAD_TIMEOUT)
 	defer cancel()
 
-	fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader([]byte(contents)))
+	fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader(contents))
 	if err != nil {
 		log.Printf("%v", err)
 		return cid.Undef, err
@@ -285,11 +292,11 @@ func cidHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) {
 	return fcid, nil
 }
 
-func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) {
+func uploadHandler(contents []byte, node pinner.PinnerNode) (cid.Cid, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), UPLOAD_TIMEOUT)
 	defer cancel()
 
-	fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader([]byte(contents)))
+	fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader(contents))
 	if err != nil {
 		log.Printf("%v", err)
 		return cid.Undef, err