Merge pull request #1010 from ipfs/fix/pin-bug

fix pinning

cmd/ipfs/init.go

@@ -143,7 +143,7 @@ func addDefaultAssets(out io.Writer, repoRoot string) error {
 		return err
 	}
 
-	if err := nd.Pinning.Pin(dir, true); err != nil {
+	if err := nd.Pinning.Pin(ctx, dir, true); err != nil {
 		return err
 	}
 

core/commands/add.go

@@ -1,25 +1,24 @@
 package commands
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"path"
 	"strings"
 
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
 	cmds "github.com/ipfs/go-ipfs/commands"
 	files "github.com/ipfs/go-ipfs/commands/files"
 	core "github.com/ipfs/go-ipfs/core"
 	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	importer "github.com/ipfs/go-ipfs/importer"
 	"github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
-	pinning "github.com/ipfs/go-ipfs/pin"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	u "github.com/ipfs/go-ipfs/util"
 	"github.com/ipfs/go-ipfs/util/debugerror"
-
-	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
 )
 
 // Error indicating the max depth has been exceded.
@@ -105,7 +104,19 @@ remains to be implemented.
 					return
 				}
 
-				_, err = addFile(n, file, outChan, progress, wrap)
+				rootnd, err := addFile(n, file, outChan, progress, wrap)
+				if err != nil {
+					res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
+					return
+				}
+
+				err = n.Pinning.Pin(context.Background(), rootnd, true)
+				if err != nil {
+					res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
+					return
+				}
+
+				err = n.Pinning.Flush()
 				if err != nil {
 					res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
 					return
@@ -200,15 +211,10 @@ remains to be implemented.
 }
 
 func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
-	mp, ok := n.Pinning.(pinning.ManualPinner)
-	if !ok {
-		return nil, errors.New("invalid pinner type! expected manual pinner")
-	}
-
 	dagnodes := make([]*dag.Node, 0)
 
 	for _, reader := range readers {
-		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
+		node, err := importer.BuildDagFromReader(reader, n.DAG, nil, chunk.DefaultSplitter)
 		if err != nil {
 			return nil, err
 		}
@@ -229,11 +235,6 @@ func addNode(n *core.IpfsNode, node *dag.Node) error {
 		return err
 	}
 
-	err = n.Pinning.Pin(node, true) // ensure we keep it
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

core/commands/ls.go

@@ -5,6 +5,9 @@ import (
 	"fmt"
 	"io"
 	"text/tabwriter"
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
 	merkledag "github.com/ipfs/go-ipfs/merkledag"
@@ -77,7 +80,9 @@ it contains, with the following format:
 				Links: make([]LsLink, len(dagnode.Links)),
 			}
 			for j, link := range dagnode.Links {
-				link.Node, err = link.GetNode(node.DAG)
+				ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+				defer cancel()
+				link.Node, err = link.GetNode(ctx, node.DAG)
 				if err != nil {
 					res.SetError(err, cmds.ErrNormal)
 					return

core/commands/pin.go

@@ -165,12 +165,14 @@ Use --type=<type> to specify the type of pinned keys to list. Valid values are:
     * "indirect": pinned indirectly by an ancestor (like a refcount)
     * "all"
 
+To see the ref count on indirect pins, pass the -count option flag.
 Defaults to "direct".
 `,
 	},
 
 	Options: []cmds.Option{
 		cmds.StringOption("type", "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\". Defaults to \"direct\""),
+		cmds.BoolOption("count", "n", "Show refcount when listing indirect pins"),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		n, err := req.Context().GetNode()
@@ -195,21 +197,57 @@ Defaults to "direct".
 			res.SetError(err, cmds.ErrClient)
 		}
 
-		keys := make([]u.Key, 0)
+		keys := make(map[string]int)
 		if typeStr == "direct" || typeStr == "all" {
-			keys = append(keys, n.Pinning.DirectKeys()...)
+			for _, k := range n.Pinning.DirectKeys() {
+				keys[k.B58String()] = 1
+			}
 		}
 		if typeStr == "indirect" || typeStr == "all" {
-			keys = append(keys, n.Pinning.IndirectKeys()...)
+			for k, v := range n.Pinning.IndirectKeys() {
+				keys[k.B58String()] = v
+			}
 		}
 		if typeStr == "recursive" || typeStr == "all" {
-			keys = append(keys, n.Pinning.RecursiveKeys()...)
+			for _, k := range n.Pinning.RecursiveKeys() {
+				keys[k.B58String()] = 1
+			}
 		}
 
-		res.SetOutput(&KeyList{Keys: keys})
+		res.SetOutput(&RefKeyList{Keys: keys})
 	},
-	Type: KeyList{},
+	Type: RefKeyList{},
 	Marshalers: cmds.MarshalerMap{
-		cmds.Text: KeyListTextMarshaler,
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			typeStr, _, err := res.Request().Option("type").String()
+			if err != nil {
+				return nil, err
+			}
+
+			count, _, err := res.Request().Option("count").Bool()
+			if err != nil {
+				return nil, err
+			}
+
+			keys, ok := res.Output().(*RefKeyList)
+			if !ok {
+				return nil, u.ErrCast()
+			}
+			out := new(bytes.Buffer)
+			if typeStr == "indirect" && count {
+				for k, v := range keys.Keys {
+					fmt.Fprintf(out, "%s %d\n", k, v)
+				}
+			} else {
+				for k, _ := range keys.Keys {
+					fmt.Fprintf(out, "%s\n", k)
+				}
+			}
+			return out, nil
+		},
 	},
 }
+
+type RefKeyList struct {
+	Keys map[string]int
+}

core/corehttp/gateway_handler.go

@@ -352,7 +352,9 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	rootnd, err := i.node.Resolver.DAG.Get(u.Key(h))
+	tctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+	rootnd, err := i.node.Resolver.DAG.Get(tctx, u.Key(h))
 	if err != nil {
 		webError(w, "Could not resolve root object", err, http.StatusBadRequest)
 		return
@@ -414,7 +416,9 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	rootnd, err := i.node.Resolver.DAG.Get(u.Key(h))
+	tctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+	rootnd, err := i.node.Resolver.DAG.Get(tctx, u.Key(h))
 	if err != nil {
 		webError(w, "Could not resolve root object", err, http.StatusBadRequest)
 		return

core/corerepo/pinning.go

@@ -2,6 +2,9 @@ package corerepo
 
 import (
 	"fmt"
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/merkledag"
@@ -27,7 +30,9 @@ func Pin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {
 			return nil, err
 		}
 
-		err = n.Pinning.Pin(dagnode, recursive)
+		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+		defer cancel()
+		err = n.Pinning.Pin(ctx, dagnode, recursive)
 		if err != nil {
 			return nil, fmt.Errorf("pin: %s", err)
 		}
@@ -56,7 +61,10 @@ func Unpin(n *core.IpfsNode, paths []string, recursive bool) ([]u.Key, error) {
 	var unpinned []u.Key
 	for _, dagnode := range dagnodes {
 		k, _ := dagnode.Key()
-		err := n.Pinning.Unpin(k, recursive)
+
+		ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+		defer cancel()
+		err := n.Pinning.Unpin(ctx, k, recursive)
 		if err != nil {
 			return nil, err
 		}

core/coreunix/add.go

@@ -6,6 +6,9 @@ import (
 	"io/ioutil"
 	"os"
 	gopath "path"
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/ipfs/go-ipfs/commands/files"
 	core "github.com/ipfs/go-ipfs/core"
@@ -108,7 +111,9 @@ func addNode(n *core.IpfsNode, node *merkledag.Node) error {
 	if err != nil {
 		return err
 	}
-	err = n.Pinning.Pin(node, true) // ensure we keep it
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+	defer cancel()
+	err = n.Pinning.Pin(ctx, node, true) // ensure we keep it
 	if err != nil {
 		return err
 	}

core/coreunix/metadata.go

@@ -1,6 +1,10 @@
 package coreunix
 
 import (
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
 	core "github.com/ipfs/go-ipfs/core"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
@@ -9,7 +13,10 @@ import (
 
 func AddMetadataTo(n *core.IpfsNode, key string, m *ft.Metadata) (string, error) {
 	ukey := u.B58KeyDecode(key)
-	nd, err := n.DAG.Get(ukey)
+
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+	defer cancel()
+	nd, err := n.DAG.Get(ctx, ukey)
 	if err != nil {
 		return "", err
 	}
@@ -36,7 +43,10 @@ func AddMetadataTo(n *core.IpfsNode, key string, m *ft.Metadata) (string, error)
 
 func Metadata(n *core.IpfsNode, key string) (*ft.Metadata, error) {
 	ukey := u.B58KeyDecode(key)
-	nd, err := n.DAG.Get(ukey)
+
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+	defer cancel()
+	nd, err := n.DAG.Get(ctx, ukey)
 	if err != nil {
 		return nil, err
 	}

core/coreunix/metadata_test.go

@@ -8,6 +8,7 @@ import (
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	dssync "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
 	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
 	bserv "github.com/ipfs/go-ipfs/blockservice"
 	core "github.com/ipfs/go-ipfs/core"
@@ -65,7 +66,7 @@ func TestMetadata(t *testing.T) {
 		t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
 	}
 
-	retnode, err := ds.Get(u.B58KeyDecode(mdk))
+	retnode, err := ds.Get(context.Background(), u.B58KeyDecode(mdk))
 	if err != nil {
 		t.Fatal(err)
 	}

fuse/ipns/common.go

@@ -1,6 +1,10 @@
 package ipns
 
 import (
+	"time"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
 	"github.com/ipfs/go-ipfs/core"
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 	nsys "github.com/ipfs/go-ipfs/namesys"
@@ -17,7 +21,10 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
 		return err
 	}
 
-	err = n.Pinning.Pin(emptyDir, false)
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+	defer cancel()
+
+	err = n.Pinning.Pin(ctx, emptyDir, false)
 	if err != nil {
 		return err
 	}

fuse/readonly/ipfs_test.go

@@ -93,7 +93,7 @@ func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.Node) []str
 	}
 	var out []string
 	for _, lnk := range n.Links {
-		child, err := lnk.GetNode(ipfs.DAG)
+		child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
 		if err != nil {
 			t.Fatal(err)
 		}

importer/helpers/helpers.go

@@ -2,7 +2,9 @@ package helpers
 
 import (
 	"fmt"
+	"time"
 
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	"github.com/ipfs/go-ipfs/pin"
@@ -76,7 +78,10 @@ func (n *UnixfsNode) NumChildren() int {
 }
 
 func (n *UnixfsNode) GetChild(i int, ds dag.DAGService) (*UnixfsNode, error) {
-	nd, err := n.node.Links[i].GetNode(ds)
+	ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
+	defer cancel()
+
+	nd, err := n.node.Links[i].GetNode(ctx, ds)
 	if err != nil {
 		return nil, err
 	}

importer/trickle/trickle_test.go

@@ -607,7 +607,7 @@ func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) {
 		fmt.Println()
 	}
 	for _, lnk := range nd.Links {
-		child, err := lnk.GetNode(ds)
+		child, err := lnk.GetNode(context.Background(), ds)
 		if err != nil {
 			panic(err)
 		}