add --enable-gc flag

README.md

@@ -66,6 +66,12 @@ NOTE: If you want more control over CLI params, you can run the server binary (a
 
 NOTE: If you get some error when running this, check if the diagnostic is there in [known issues](#known-issues)
 
+NOTE: If you want to run the server with garbage collection enabled, you can run the server binary with the `--enable-gc` flag:
+
+```bash
+./build/bin/server -w3-agent-key <AGENT_KEY> -w3-delegation-file <DELEGATION_PROOF_FILE_PATH> --enable-gc
+```
+
 ipfs-pinner can be run as a server and allows two functionalities currently - `/get` and `/upload`
 
 ### Upload a file

core/ipfs.go

@@ -15,7 +15,7 @@ import (
 )
 
 // returns a go-ipfs node backend CoreAPI instance
-func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
+func CreateIpfsNode(ctx context.Context, cidComputeOnly bool) (*core.IpfsNode, error) {
 	cfg := core.BuildCfg{
 		Online:    !cidComputeOnly, // networking
 		Permanent: !cidComputeOnly, // data persists across restarts?
@@ -50,7 +50,7 @@ func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
 	}
 
 	var nnode *core.IpfsNode
-	if nnode, err = core.NewNode(context.Background(), &cfg); err != nil {
+	if nnode, err = core.NewNode(ctx, &cfg); err != nil {
 		return nil, err
 	}
 

coreapi/api.go

@@ -15,4 +15,5 @@ type CoreExtensionAPI interface {
 
 type GarbageCollectAPI interface {
 	GarbageCollect(ctx context.Context)
+	InitPeriodicGC(ctx context.Context) <-chan error
 }

coreapi/impl.go

@@ -43,3 +43,12 @@ func (gci *garbageCollectorImpl) GarbageCollect(ctx context.Context) {
 		log.Println("error getting garbage collector: %w", err)
 	}
 }
+
+func (gci *garbageCollectorImpl) InitPeriodicGC(ctx context.Context) <-chan error {
+	errc := make(chan error)
+	go func() {
+		errc <- corerepo.PeriodicGC(ctx, gci.node)
+		close(errc)
+	}()
+	return errc
+}

pinclient/client_create_request.go

@@ -16,6 +16,7 @@ type ClientCreateRequest struct {
 	W3_AgentKey            string
 	W3_AgentDid            did.DID
 	W3_DelegationProofPath string
+	GC_Enable              bool
 
 	httpClient *http.Client
 }
@@ -56,3 +57,8 @@ func (r ClientCreateRequest) HttpClient(client http.Client) ClientCreateRequest
 	r.httpClient = &client
 	return r
 }
+
+func (r ClientCreateRequest) GcEnable(gcEnable bool) ClientCreateRequest {
+	r.GC_Enable = gcEnable
+	return r
+}

pinner.go

@@ -5,6 +5,7 @@
 package pinner
 
 import (
+	"context"
 	"log"
 
 	car "github.com/covalenthq/ipfs-pinner/car"
@@ -22,14 +23,18 @@ type pinnerNode struct {
 	pinApiClient  pinclient.PinServiceAPI
 }
 
-func NewPinnerNode(req PinnerNodeCreateRequest) PinnerNode {
+func NewPinnerNode(ctx context.Context, req PinnerNodeCreateRequest) PinnerNode {
 	node := pinnerNode{}
-	ipfsNode, err := core.CreateIpfsNode(req.cidComputeOnly)
+	ipfsNode, err := core.CreateIpfsNode(ctx, req.cidComputeOnly)
 	if err != nil {
 		log.Fatal("error initializing ipfs node: ", err)
 	}
 
 	node.ipfsCore = coreapi.NewCoreExtensionApi(ipfsNode)
+	if req.enableGC {
+		log.Print("enabling garbage collection....")
+		node.ipfsCore.GC().InitPeriodicGC(ctx)
+	}
 
 	//SETUP W3UP
 	log.Print("setting up w3up for uploads....")

pinner_create_request.go

@@ -9,14 +9,16 @@ type PinnerNodeCreateRequest struct {
 	cidComputeOnly    bool
 	cidVersion        int
 	ipfsFetchUrls     []string
+	enableGC          bool
 }
 
-func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string) *PinnerNodeCreateRequest {
+func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string, enableGC bool) *PinnerNodeCreateRequest {
 	request := new(PinnerNodeCreateRequest)
 	request.cidVersion = 0
 	request.cidComputeOnly = true
 	request.pinServiceRequest = clientRequest
 	request.ipfsFetchUrls = ipfsFetchUrls
+	request.enableGC = enableGC
 	return request
 }
 

server/main.go

@@ -45,6 +45,11 @@ type Config struct {
 	w3AgentDid          did.DID
 	delegationProofPath string
 	ipfsGatewayUrls     []string
+	enableGC            bool
+}
+
+func NewConfig(portNumber int, w3AgentKey string, w3AgentDid did.DID, delegationProofPath string, ipfsGatewayUrls []string, enableGC bool) *Config {
+	return &Config{portNumber, w3AgentKey, w3AgentDid, delegationProofPath, ipfsGatewayUrls, enableGC}
 }
 
 var (
@@ -62,6 +67,8 @@ func main() {
 
 	ipfsGatewayUrls := flag.String("ipfs-gateway-urls", "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s", "comma separated list of ipfs gateway urls")
 
+	enableGC := flag.Bool("enable-gc", false, "enable garbage collection")
+
 	flag.Parse()
 	core.Version()
 
@@ -85,17 +92,25 @@ func main() {
 
 	log.Printf("agent did: %s", agentSigner.DID().DID().String())
 
-	setUpAndRunServer(Config{*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ",")})
+	config := NewConfig(*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ","), *enableGC)
+
+	setUpAndRunServer(*config)
 }
 
 func setUpAndRunServer(config Config) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	mux := http.NewServeMux()
 	httpState := NewState()
 
-	clientCreateReq := client.NewClientRequest(core.Web3Storage).W3AgentKey(config.w3AgentKey).W3AgentDid(config.w3AgentDid).DelegationProofPath(config.delegationProofPath)
+	clientCreateReq := client.NewClientRequest(core.Web3Storage).
+		W3AgentKey(config.w3AgentKey).
+		W3AgentDid(config.w3AgentDid).
+		DelegationProofPath(config.delegationProofPath).
+		GcEnable(config.enableGC)
 	// check if cid compute true works with car uploads
-	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls).CidVersion(1).CidComputeOnly(false)
-	node := pinner.NewPinnerNode(*nodeCreateReq)
+	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls, config.enableGC).CidVersion(1).CidComputeOnly(false)
+	node := pinner.NewPinnerNode(ctx, *nodeCreateReq)
 
 	mux.Handle("/upload", recoveryWrapper(uploadHttpHandler(node)))
 	mux.Handle("/get", recoveryWrapper(downloadHttpHandler(node)))
@@ -319,19 +334,6 @@ func uploadHandler(contents string, node pinner.PinnerNode) (cid.Cid, error) {
 	log.Printf("uploaded file has root cid: %s\n", ccid)
 
 	carf.Close()
-
-	assertEquals(fcid, ccid)
-	log.Printf("the two cids match: %s\n", ccid.String())
-
-	log.Printf("removing dag...")
-	curr := time.Now().UnixMilli()
-	err = node.UnixfsService().RemoveDag(ctx, ccid)
-	after := time.Now().UnixMilli()
-	log.Println("time taken:", after-curr)
-	if err != nil {
-		log.Fatalf("%v", err)
-	}
-
 	return ccid, nil
 }
 
@@ -406,9 +408,3 @@ func timeoutHttpHandler(s *State) http.Handler {
 	}
 	return http.HandlerFunc(fn)
 }
-
-func assertEquals(obj1 interface{}, obj2 interface{}) {
-	if obj1 != obj2 {
-		log.Fatalf("fail %v and %v doesn't match", obj1, obj2)
-	}
-}