Named pins, storing pins in Datastore

* Change pin structure to a tree stored in KV store
* Adjust pin-related commands accordingly
* Make pinning routines use meaningful default prefixes

License: MIT
Signed-off-by: Iaroslav Gridin <[email protected]>

Author: Voker57

assets/assets.go

@@ -80,13 +80,9 @@ func addAssetList(nd *core.IpfsNode, l []string) (cid.Cid, error) {
 		return cid.Cid{}, err
 	}
 
-	if err := nd.Pinning.Pin(nd.Context(), dir, true); err != nil {
+	if err := nd.Pinning.Pin(nd.Context(), "assets", dir, true); err != nil {
 		return cid.Cid{}, fmt.Errorf("assets: Pinning on init-docu failed: %s", err)
 	}
 
-	if err := nd.Pinning.Flush(); err != nil {
-		return cid.Cid{}, fmt.Errorf("assets: Pinning flush failed: %s", err)
-	}
-
 	return dir.Cid(), nil
 }

core/builder.go

@@ -268,15 +268,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
 	n.Blocks = bserv.New(n.Blockstore, n.Exchange)
 	n.DAG = dag.NewDAGService(n.Blocks)
 
-	internalDag := dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore)))
-	n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG, internalDag)
-	if err != nil {
-		// TODO: we should move towards only running 'NewPinner' explicitly on
-		// node init instead of implicitly here as a result of the pinner keys
-		// not being found in the datastore.
-		// this is kinda sketchy and could cause data loss
-		n.Pinning = pin.NewPinner(n.Repo.Datastore(), n.DAG, internalDag)
-	}
+	n.Pinning = pin.NewPinner(n.DAG, n.Repo.Datastore())
 	n.Resolver = resolver.NewBasicResolver(n.DAG)
 
 	if cfg.Online {

core/commands/add.go

@@ -39,6 +39,7 @@ const (
 	onlyHashOptionName    = "only-hash"
 	chunkerOptionName     = "chunker"
 	pinOptionName         = "pin"
+	pinPathOptionName     = "pinpath"
 	rawLeavesOptionName   = "raw-leaves"
 	noCopyOptionName      = "nocopy"
 	fstoreCacheOptionName = "fscache"
@@ -119,6 +120,7 @@ You can now check what blocks have been created by:
 		cmdkit.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
 		cmdkit.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes] or rabin-[min]-[avg]-[max]").WithDefault("size-262144"),
 		cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
+		cmdkit.StringOption(pinPathOptionName, "P", "Pin object under this path.").WithDefault("added/"),
 		cmdkit.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
 		cmdkit.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
@@ -160,6 +162,7 @@ You can now check what blocks have been created by:
 		silent, _ := req.Options[silentOptionName].(bool)
 		chunker, _ := req.Options[chunkerOptionName].(string)
 		dopin, _ := req.Options[pinOptionName].(bool)
+		pinPath, _ := req.Options[pinPathOptionName].(string)
 		rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
 		nocopy, _ := req.Options[noCopyOptionName].(bool)
 		fscache, _ := req.Options[fstoreCacheOptionName].(bool)
@@ -185,6 +188,7 @@ You can now check what blocks have been created by:
 			options.Unixfs.Chunker(chunker),
 
 			options.Unixfs.Pin(dopin),
+			options.Unixfs.PinPath(pinPath),
 			options.Unixfs.HashOnly(hash),
 			options.Unixfs.FsCache(fscache),
 			options.Unixfs.Nocopy(nocopy),

core/commands/dag/dag.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	"github.com/ipfs/go-ipfs/core/coredag"
-	"github.com/ipfs/go-ipfs/pin"
 
 	path "gx/ipfs/QmNYPETsdAu2uQ1k9q9S1jYEGURaLHV6cbYRSVFVRftpF8/go-path"
 	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
@@ -60,7 +59,7 @@ into an object of the specified format.
 	Options: []cmdkit.Option{
 		cmdkit.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"),
 		cmdkit.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"),
-		cmdkit.BoolOption("pin", "Pin this object when adding."),
+		cmdkit.StringOption("pin", "Pin this object when adding.").WithDefault(""),
 		cmdkit.StringOption("hash", "Hash function to use").WithDefault(""),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -72,7 +71,7 @@ into an object of the specified format.
 		ienc, _ := req.Options["input-enc"].(string)
 		format, _ := req.Options["format"].(string)
 		hash, _ := req.Options["hash"].(string)
-		dopin, _ := req.Options["pin"].(bool)
+		pinpath, _ := req.Options["pin"].(string)
 
 		// mhType tells inputParser which hash should be used. MaxUint64 means 'use
 		// default hash' (sha256 for cbor, sha1 for git..)
@@ -89,10 +88,6 @@ into an object of the specified format.
 		cids := cid.NewSet()
 		b := ipld.NewBatch(req.Context, nd.DAG)
 
-		if dopin {
-			defer nd.Blockstore.PinLock().Unlock()
-		}
-
 		it := req.Files.Entries()
 		for it.Next() {
 			file := files.FileFromEntry(it)
@@ -128,13 +123,11 @@ into an object of the specified format.
 			return err
 		}
 
-		if dopin {
-			cids.ForEach(func(c cid.Cid) error {
-				nd.Pinning.PinWithMode(c, pin.Recursive)
-				return nil
+		if pinpath != "" {
+			err := cids.ForEach(func(c cid.Cid) error {
+				return nd.Pinning.AddPin(pinpath, c, true)
 			})
 
-			err := nd.Pinning.Flush()
 			if err != nil {
 				return err
 			}

core/commands/object/object.go

@@ -383,6 +383,7 @@ And then run:
 		cmdkit.StringOption("inputenc", "Encoding type of input data. One of: {\"protobuf\", \"json\"}.").WithDefault("json"),
 		cmdkit.StringOption("datafieldenc", "Encoding type of the data field, either \"text\" or \"base64\".").WithDefault("text"),
 		cmdkit.BoolOption("pin", "Pin this object when adding."),
+		cmdkit.StringOption("pinpath", "Pin under this path").WithDefault("added/"),
 		cmdkit.BoolOption("quiet", "q", "Write minimal output."),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -406,15 +407,21 @@ And then run:
 			return err
 		}
 
-		dopin, _ := req.Options["pin"].(bool)
+		pinpath, _ := req.Options["pinpath"].(string)
+		if err != nil {
+			return err
+		}
+
+		pin, _ := req.Options["pin"].(bool)
 		if err != nil {
 			return err
 		}
 
 		p, err := api.Object().Put(req.Context, file,
 			options.Object.DataType(datafieldenc),
 			options.Object.InputEnc(inputenc),
-			options.Object.Pin(dopin))
+			options.Object.Pin(pin),
+			options.Object.PinPath(pinpath))
 		if err != nil {
 			return err
 		}