From 84882735f2b88c788897bafb3f49f67623bc69eb Mon Sep 17 00:00:00 2001 From: tarassh Date: Sat, 16 Mar 2024 17:53:57 +0000 Subject: [PATCH] add `--enable-gc` flag --- README.md | 6 +++++ core/ipfs.go | 4 +-- coreapi/api.go | 1 + coreapi/impl.go | 9 +++++++ pinclient/client_create_request.go | 6 +++++ pinner.go | 9 +++++-- pinner_create_request.go | 4 ++- server/main.go | 42 ++++++++++++++---------------- 8 files changed, 53 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9bf72b3..60e8048 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ NOTE: If you want more control over CLI params, you can run the server binary (a NOTE: If you get some error when running this, check if the diagnostic is there in [known issues](#known-issues) +NOTE: If you want to run the server with garbage collection enabled, you can run the server binary with the `--enable-gc` flag: + +```bash +./build/bin/server -w3-agent-key -w3-delegation-file --enable-gc +``` + ipfs-pinner can be run as a server and allows two functionalities currently - `/get` and `/upload` ### Upload a file diff --git a/core/ipfs.go b/core/ipfs.go index ac31929..b290b8b 100644 --- a/core/ipfs.go +++ b/core/ipfs.go @@ -15,7 +15,7 @@ import ( ) // returns a go-ipfs node backend CoreAPI instance -func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) { +func CreateIpfsNode(ctx context.Context, cidComputeOnly bool) (*core.IpfsNode, error) { cfg := core.BuildCfg{ Online: !cidComputeOnly, // networking Permanent: !cidComputeOnly, // data persists across restarts? @@ -50,7 +50,7 @@ func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) { } var nnode *core.IpfsNode - if nnode, err = core.NewNode(context.Background(), &cfg); err != nil { + if nnode, err = core.NewNode(ctx, &cfg); err != nil { return nil, err } diff --git a/coreapi/api.go b/coreapi/api.go index 6439582..b0b9092 100644 --- a/coreapi/api.go +++ b/coreapi/api.go @@ -15,4 +15,5 @@ type CoreExtensionAPI interface { type GarbageCollectAPI interface { GarbageCollect(ctx context.Context) + InitPeriodicGC(ctx context.Context) <-chan error } diff --git a/coreapi/impl.go b/coreapi/impl.go index 3f261e6..508f451 100644 --- a/coreapi/impl.go +++ b/coreapi/impl.go @@ -43,3 +43,12 @@ func (gci *garbageCollectorImpl) GarbageCollect(ctx context.Context) { log.Println("error getting garbage collector: %w", err) } } + +func (gci *garbageCollectorImpl) InitPeriodicGC(ctx context.Context) <-chan error { + errc := make(chan error) + go func() { + errc <- corerepo.PeriodicGC(ctx, gci.node) + close(errc) + }() + return errc +} diff --git a/pinclient/client_create_request.go b/pinclient/client_create_request.go index bb3d041..51642ed 100644 --- a/pinclient/client_create_request.go +++ b/pinclient/client_create_request.go @@ -16,6 +16,7 @@ type ClientCreateRequest struct { W3_AgentKey string W3_AgentDid did.DID W3_DelegationProofPath string + GC_Enable bool httpClient *http.Client } @@ -56,3 +57,8 @@ func (r ClientCreateRequest) HttpClient(client http.Client) ClientCreateRequest r.httpClient = &client return r } + +func (r ClientCreateRequest) GcEnable(gcEnable bool) ClientCreateRequest { + r.GC_Enable = gcEnable + return r +} diff --git a/pinner.go b/pinner.go index ae514d4..4c5d852 100644 --- a/pinner.go +++ b/pinner.go @@ -5,6 +5,7 @@ package pinner import ( + "context" "log" car "github.com/covalenthq/ipfs-pinner/car" @@ -22,14 +23,18 @@ type pinnerNode struct { pinApiClient pinclient.PinServiceAPI } -func NewPinnerNode(req PinnerNodeCreateRequest) PinnerNode { +func NewPinnerNode(ctx context.Context, req PinnerNodeCreateRequest) PinnerNode { node := pinnerNode{} - ipfsNode, err := core.CreateIpfsNode(req.cidComputeOnly) + ipfsNode, err := core.CreateIpfsNode(ctx, req.cidComputeOnly) if err != nil { log.Fatal("error initializing ipfs node: ", err) } node.ipfsCore = coreapi.NewCoreExtensionApi(ipfsNode) + if req.enableGC { + log.Print("enabling garbage collection....") + node.ipfsCore.GC().InitPeriodicGC(ctx) + } //SETUP W3UP log.Print("setting up w3up for uploads....") diff --git a/pinner_create_request.go b/pinner_create_request.go index 4b8b271..d8bc5c7 100644 --- a/pinner_create_request.go +++ b/pinner_create_request.go @@ -9,14 +9,16 @@ type PinnerNodeCreateRequest struct { cidComputeOnly bool cidVersion int ipfsFetchUrls []string + enableGC bool } -func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string) *PinnerNodeCreateRequest { +func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string, enableGC bool) *PinnerNodeCreateRequest { request := new(PinnerNodeCreateRequest) request.cidVersion = 0 request.cidComputeOnly = true request.pinServiceRequest = clientRequest request.ipfsFetchUrls = ipfsFetchUrls + request.enableGC = enableGC return request } diff --git a/server/main.go b/server/main.go index 0efb75b..fd0ea79 100644 --- a/server/main.go +++ b/server/main.go @@ -45,6 +45,11 @@ type Config struct { w3AgentDid did.DID delegationProofPath string ipfsGatewayUrls []string + enableGC bool +} + +func NewConfig(portNumber int, w3AgentKey string, w3AgentDid did.DID, delegationProofPath string, ipfsGatewayUrls []string, enableGC bool) *Config { + return &Config{portNumber, w3AgentKey, w3AgentDid, delegationProofPath, ipfsGatewayUrls, enableGC} } var ( @@ -62,6 +67,8 @@ func main() { ipfsGatewayUrls := flag.String("ipfs-gateway-urls", "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s", "comma separated list of ipfs gateway urls") + enableGC := flag.Bool("enable-gc", false, "enable garbage collection") + flag.Parse() core.Version() @@ -85,17 +92,25 @@ func main() { log.Printf("agent did: %s", agentSigner.DID().DID().String()) - setUpAndRunServer(Config{*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ",")}) + config := NewConfig(*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ","), *enableGC) + + setUpAndRunServer(*config) } func setUpAndRunServer(config Config) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() mux := http.NewServeMux() httpState := NewState() - clientCreateReq := client.NewClientRequest(core.Web3Storage).W3AgentKey(config.w3AgentKey).W3AgentDid(config.w3AgentDid).DelegationProofPath(config.delegationProofPath) + clientCreateReq := client.NewClientRequest(core.Web3Storage). + W3AgentKey(config.w3AgentKey). + W3AgentDid(config.w3AgentDid). + DelegationProofPath(config.delegationProofPath). + GcEnable(config.enableGC) // check if cid compute true works with car uploads - nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls).CidVersion(1).CidComputeOnly(false) - node := pinner.NewPinnerNode(*nodeCreateReq) + nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls, config.enableGC).CidVersion(1).CidComputeOnly(false) + node := pinner.NewPinnerNode(ctx, *nodeCreateReq) mux.Handle("/upload", recoveryWrapper(uploadHttpHandler(node))) mux.Handle("/get", recoveryWrapper(downloadHttpHandler(node))) @@ -319,19 +334,6 @@ func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) { log.Printf("uploaded file has root cid: %s\n", ccid) carf.Close() - - assertEquals(fcid, ccid) - log.Printf("the two cids match: %s\n", ccid.String()) - - log.Printf("removing dag...") - curr := time.Now().UnixMilli() - err = node.UnixfsService().RemoveDag(ctx, ccid) - after := time.Now().UnixMilli() - log.Println("time taken:", after-curr) - if err != nil { - log.Fatalf("%v", err) - } - return ccid, nil } @@ -406,9 +408,3 @@ func timeoutHttpHandler(s *State) http.Handler { } return http.HandlerFunc(fn) } - -func assertEquals(obj1 interface{}, obj2 interface{}) { - if obj1 != obj2 { - log.Fatalf("fail %v and %v doesn't match", obj1, obj2) - } -}