diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 978616c..d714c9a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,5 +12,5 @@ updates: target-branch: "develop" reviewers: - "noslav" - - "sudeepdino008" + - "rogarcia" diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index 5cd6cfd..233fdc0 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -3,16 +3,20 @@ name: docker-ci on: push: branches: - - "main" + - "main" pull_request: - branches: - - "main" - - "develop" + branches: + - "main" + - "develop" jobs: build: runs-on: ubuntu-latest steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} - name: Login to Artifact Registry/GCR uses: docker/login-action@v2 with: diff --git a/core/support.go b/core/support.go index 2c931f2..6d3d8de 100644 --- a/core/support.go +++ b/core/support.go @@ -17,7 +17,7 @@ const ( // IpfsPinnerVersionMinor is Minor version component of the current release IpfsPinnerVersionMinor = 2 // IpfsPinnerVersionPatch is Patch version component of the current release - IpfsPinnerVersionPatch = 1 + IpfsPinnerVersionPatch = 3 clientIdentifier = "ipfs-pinner" // Client identifier to advertise over the network ) diff --git a/go.mod b/go.mod index 3c8e1a8..25d9a6c 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,13 @@ require ( github.com/ipfs/kubo v0.29.0 github.com/ipld/go-car v0.6.2 github.com/ipld/go-ipld-prime v0.21.0 - github.com/multiformats/go-multiaddr v0.12.4 + github.com/multiformats/go-multiaddr v0.14.0 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multihash v0.2.3 github.com/pkg/errors v0.9.1 github.com/web3-storage/go-ucanto v0.1.0 github.com/ybbus/httpretry v1.0.2 - golang.org/x/oauth2 v0.21.0 + golang.org/x/oauth2 v0.24.0 ) require ( diff --git a/go.sum b/go.sum index 2bab65e..f90611a 100644 --- a/go.sum +++ b/go.sum @@ -519,8 +519,8 @@ github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= -github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc= -github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= +github.com/multiformats/go-multiaddr v0.14.0 h1:bfrHrJhrRuh/NXH5mCnemjpbGjzRw/b+tJFOD41g2tU= +github.com/multiformats/go-multiaddr v0.14.0/go.mod h1:6EkVAxtznq2yC3QT5CM1UTAwG0GTP3EWAIcjHuzQ+r4= github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= @@ -941,8 +941,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/server/main.go b/server/main.go index 36b4430..96c8f24 100644 --- a/server/main.go +++ b/server/main.go @@ -57,6 +57,8 @@ var ( DOWNLOAD_TIMEOUT = 12 * time.Minute // download can take a lot of time if it's not locally present UPLOAD_TIMEOUT = 60 * time.Second // uploads of around 6MB files happen in less than 10s typically + + maxMemory = int64(100 << 20) // 100 MB ) func main() { @@ -166,15 +168,18 @@ func uploadHttpHandler(node pinner.PinnerNode) http.Handler { respondError(w, err) return } - ccid, err := uploadHandler(contents, node) - if err != nil { - respondError(w, err) - return - } else { - succ_str := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String()) - _, err := w.Write([]byte(succ_str)) + + for _, v := range contents { + ccid, err := uploadHandler(v, node) if err != nil { - log.Println("error writing data to connection: %w", err) + respondError(w, err) + return + } else { + succStr := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String()) + _, err := w.Write([]byte(succStr)) + if err != nil { + log.Println("error writing data to connection: %w", err) + } } } } @@ -209,15 +214,17 @@ func cidHttpHandler(node pinner.PinnerNode) http.Handler { return } - ccid, err := cidHandler(contents, node) - if err != nil { - respondError(w, err) - return - } else { - succ_str := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String()) - _, err := w.Write([]byte(succ_str)) + for _, v := range contents { + ccid, err := cidHandler(v, node) if err != nil { - log.Println("error writing data to connection: %w", err) + respondError(w, err) + return + } else { + succStr := fmt.Sprintf("{\"cid\": \"%s\"}", ccid.String()) + _, err := w.Write([]byte(succStr)) + if err != nil { + log.Println("error writing data to connection: %w", err) + } } } } @@ -225,29 +232,29 @@ func cidHttpHandler(node pinner.PinnerNode) http.Handler { return http.HandlerFunc(fn) } -func readContentFromRequest(r *http.Request) (string, error) { - mreader, err := r.MultipartReader() +func readContentFromRequest(r *http.Request) (map[string][]byte, error) { + err := r.ParseMultipartForm(maxMemory) if err != nil { - return "", err + return nil, err } - var contents string = "" - - for { - part, err := mreader.NextPart() - if err == io.EOF { - break - } + files := make(map[string][]byte) + for _, fileHeaders := range r.MultipartForm.File { + for _, fileHeader := range fileHeaders { + file, err := fileHeader.Open() + if err != nil { + return nil, err + } + defer file.Close() - pcontents, err := io.ReadAll(part) - if err != nil { - return "", err + data, err := io.ReadAll(file) + if err != nil { + return nil, err + } + files[fileHeader.Filename] = data } - - contents += string(pcontents) } - - return contents, nil + return files, nil } func recoveryWrapper(h http.Handler) http.Handler { @@ -271,11 +278,11 @@ func recoveryWrapper(h http.Handler) http.Handler { }) } -func cidHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) { +func cidHandler(contents []byte, node pinner.PinnerNode) (cid.Cid, error) { ctx, cancel := context.WithTimeout(context.Background(), UPLOAD_TIMEOUT) defer cancel() - fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader([]byte(contents))) + fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader(contents)) if err != nil { log.Printf("%v", err) return cid.Undef, err @@ -285,11 +292,11 @@ func cidHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) { return fcid, nil } -func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) { +func uploadHandler(contents []byte, node pinner.PinnerNode) (cid.Cid, error) { ctx, cancel := context.WithTimeout(context.Background(), UPLOAD_TIMEOUT) defer cancel() - fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader([]byte(contents))) + fcid, err := node.UnixfsService().GenerateDag(ctx, bytes.NewReader(contents)) if err != nil { log.Printf("%v", err) return cid.Undef, err