diff --git a/blocks/blocks.go b/blocks/blocks.go index b50d6202621..0aea933c758 100644 --- a/blocks/blocks.go +++ b/blocks/blocks.go @@ -13,6 +13,8 @@ import ( var ErrWrongHash = errors.New("data did not match given hash!") +// Block is a singular block of data in ipfs + type Block interface { RawData() []byte Cid() *cid.Cid @@ -20,7 +22,6 @@ type Block interface { Loggable() map[string]interface{} } -// Block is a singular block of data in ipfs type BasicBlock struct { cid *cid.Cid data []byte diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index 274c1ee7b4f..8c971d89a47 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -21,7 +21,9 @@ import ( var log = logging.Logger("blockstore") // BlockPrefix namespaces blockstore datastores -var BlockPrefix = ds.NewKey("blocks") +const DefaultPrefix = "/blocks" + +var blockPrefix = ds.NewKey(DefaultPrefix) var ValueTypeMismatch = errors.New("the retrieved value is not a Block") var ErrHashMismatch = errors.New("block in storage has different hash than requested") @@ -71,20 +73,23 @@ type gcBlockstore struct { } func NewBlockstore(d ds.Batching) *blockstore { + return NewBlockstoreWPrefix(d, DefaultPrefix) +} + +func NewBlockstoreWPrefix(d ds.Batching, prefix string) *blockstore { var dsb ds.Batching - dd := dsns.Wrap(d, BlockPrefix) + prefixKey := ds.NewKey(prefix) + dd := dsns.Wrap(d, prefixKey) dsb = dd return &blockstore{ datastore: dsb, + prefix: prefixKey, } } type blockstore struct { datastore ds.Batching - - lk sync.RWMutex - gcreq int32 - gcreqlk sync.Mutex + prefix ds.Key rehash bool } @@ -130,11 +135,8 @@ func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) { func (bs *blockstore) Put(block blocks.Block) error { k := dshelp.CidToDsKey(block.Cid()) - // Has is cheaper than Put, so see if we already have it - exists, err := bs.datastore.Has(k) - if err == nil && exists { - return nil // already stored. - } + // Note: The Has Check is now done by the MultiBlockstore + return bs.datastore.Put(k, block.RawData()) } @@ -145,11 +147,6 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error { } for _, b := range blocks { k := dshelp.CidToDsKey(b.Cid()) - exists, err := bs.datastore.Has(k) - if err == nil && exists { - continue - } - err = t.Put(k, b.RawData()) if err != nil { return err @@ -175,7 +172,7 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} // datastore/namespace does *NOT* fix up Query.Prefix - q.Prefix = BlockPrefix.String() + q.Prefix = bs.prefix.String() res, err := bs.datastore.Query(q) if err != nil { return nil, err diff --git a/blocks/blockstore/blockstore_test.go b/blocks/blockstore/blockstore_test.go index abe8a1a72d5..22c15d0004b 100644 --- a/blocks/blockstore/blockstore_test.go +++ b/blocks/blockstore/blockstore_test.go @@ -170,7 +170,7 @@ func TestAllKeysRespectsContext(t *testing.T) { default: } - e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()} + e := dsq.Entry{Key: blockPrefix.ChildString("foo").String()} resultChan <- dsq.Result{Entry: e} // let it go. close(resultChan) <-done // should be done now. @@ -190,7 +190,7 @@ func TestValueTypeMismatch(t *testing.T) { block := blocks.NewBlock([]byte("some data")) datastore := ds.NewMapDatastore() - k := BlockPrefix.Child(dshelp.CidToDsKey(block.Cid())) + k := blockPrefix.Child(dshelp.CidToDsKey(block.Cid())) datastore.Put(k, "data that isn't a block!") blockstore := NewBlockstore(ds_sync.MutexWrap(datastore)) diff --git a/blocks/blockstore/bloom_cache_test.go b/blocks/blockstore/bloom_cache_test.go index 72223cd44e0..0ee3a557a5c 100644 --- a/blocks/blockstore/bloom_cache_test.go +++ b/blocks/blockstore/bloom_cache_test.go @@ -104,11 +104,11 @@ func TestHasIsBloomCached(t *testing.T) { block := blocks.NewBlock([]byte("newBlock")) cachedbs.PutMany([]blocks.Block{block}) - if cacheFails != 2 { - t.Fatalf("expected two datastore hits: %d", cacheFails) + if cacheFails != 1 { + t.Fatalf("expected datastore hits: %d", cacheFails) } cachedbs.Put(block) - if cacheFails != 3 { + if cacheFails != 2 { t.Fatalf("expected datastore hit: %d", cacheFails) } diff --git a/blocks/blockstore/multi.go b/blocks/blockstore/multi.go new file mode 100644 index 00000000000..2d260b72ff6 --- /dev/null +++ b/blocks/blockstore/multi.go @@ -0,0 +1,158 @@ +package blockstore + +// A very simple multi-blockstore that analogous to a unionfs Put and +// DeleteBlock only go to the first blockstore all others are +// considered readonly. + +import ( + //"errors" + "context" + + blocks "github.com/ipfs/go-ipfs/blocks" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query" +) + +type LocateInfo struct { + Prefix string + Error error +} + +type MultiBlockstore interface { + Blockstore + GCLocker + FirstMount() Blockstore + Mounts() []string + Mount(prefix string) Blockstore + Locate(*cid.Cid) []LocateInfo +} + +type Mount struct { + Prefix string + Blocks Blockstore +} + +func NewMultiBlockstore(mounts ...Mount) *multiblockstore { + return &multiblockstore{ + mounts: mounts, + } +} + +type multiblockstore struct { + mounts []Mount + gclocker +} + +func (bs *multiblockstore) FirstMount() Blockstore { + return bs.mounts[0].Blocks +} + +func (bs *multiblockstore) Mounts() []string { + mounts := make([]string, 0, len(bs.mounts)) + for _, mnt := range bs.mounts { + mounts = append(mounts, mnt.Prefix) + } + return mounts +} + +func (bs *multiblockstore) Mount(prefix string) Blockstore { + for _, m := range bs.mounts { + if m.Prefix == prefix { + return m.Blocks + } + } + return nil +} + +func (bs *multiblockstore) DeleteBlock(key *cid.Cid) error { + return bs.mounts[0].Blocks.DeleteBlock(key) +} + +func (bs *multiblockstore) Has(c *cid.Cid) (bool, error) { + var firstErr error + for _, m := range bs.mounts { + have, err := m.Blocks.Has(c) + if have && err == nil { + return have, nil + } + if err != nil && firstErr == nil { + firstErr = err + } + } + return false, firstErr +} + +func (bs *multiblockstore) Get(c *cid.Cid) (blocks.Block, error) { + var firstErr error + for _, m := range bs.mounts { + blk, err := m.Blocks.Get(c) + if err == nil { + return blk, nil + } + if firstErr == nil || firstErr == ErrNotFound { + firstErr = err + } + } + return nil, firstErr +} + +func (bs *multiblockstore) Locate(c *cid.Cid) []LocateInfo { + res := make([]LocateInfo, 0, len(bs.mounts)) + for _, m := range bs.mounts { + _, err := m.Blocks.Get(c) + res = append(res, LocateInfo{m.Prefix, err}) + } + return res +} + +func (bs *multiblockstore) Put(blk blocks.Block) error { + // First call Has() to make sure the block doesn't exist in any of + // the sub-blockstores, otherwise we could end with data being + // duplicated in two blockstores. + exists, err := bs.Has(blk.Cid()) + if err == nil && exists { + return nil // already stored + } + return bs.mounts[0].Blocks.Put(blk) +} + +func (bs *multiblockstore) PutMany(blks []blocks.Block) error { + stilladd := make([]blocks.Block, 0, len(blks)) + // First call Has() to make sure the block doesn't exist in any of + // the sub-blockstores, otherwise we could end with data being + // duplicated in two blockstores. + for _, blk := range blks { + exists, err := bs.Has(blk.Cid()) + if err == nil && exists { + continue // already stored + } + stilladd = append(stilladd, blk) + } + if len(stilladd) == 0 { + return nil + } + return bs.mounts[0].Blocks.PutMany(stilladd) +} + +func (bs *multiblockstore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) { + //return bs.mounts[0].Blocks.AllKeysChan(ctx) + //return nil, errors.New("Unimplemented") + in := make([]<-chan *cid.Cid, 0, len(bs.mounts)) + for _, m := range bs.mounts { + ch, err := m.Blocks.AllKeysChan(ctx) + if err != nil { + return nil, err + } + in = append(in, ch) + } + out := make(chan *cid.Cid, dsq.KeysOnlyBufSize) + go func() { + defer close(out) + for _, in0 := range in { + for key := range in0 { + out <- key + } + } + }() + return out, nil +} diff --git a/blocks/blockstore/util/remove.go b/blocks/blockstore/util/remove.go index 01f2ce44e31..7c72c29d9d3 100644 --- a/blocks/blockstore/util/remove.go +++ b/blocks/blockstore/util/remove.go @@ -27,14 +27,23 @@ type RmBlocksOpts struct { Force bool } -func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, opts RmBlocksOpts) error { +func RmBlocks(mbs bs.MultiBlockstore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, opts RmBlocksOpts) error { + prefix := opts.Prefix + if prefix == "" { + prefix = mbs.Mounts()[0] + } + blocks := mbs.Mount(prefix) + if blocks == nil { + return fmt.Errorf("Could not find blockstore: %s\n", prefix) + } + go func() { defer close(out) - unlocker := blocks.GCLock() + unlocker := mbs.GCLock() defer unlocker.Unlock() - stillOkay := FilterPinned(pins, out, cids) + stillOkay := FilterPinned(mbs, pins, out, cids, prefix) for _, c := range stillOkay { err := blocks.DeleteBlock(c) @@ -50,7 +59,7 @@ func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, c return nil } -func FilterPinned(pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid) []*cid.Cid { +func FilterPinned(mbs bs.MultiBlockstore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, prefix string) []*cid.Cid { stillOkay := make([]*cid.Cid, 0, len(cids)) res, err := pins.CheckIfPinned(cids...) if err != nil { @@ -58,7 +67,7 @@ func FilterPinned(pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid) []*c return nil } for _, r := range res { - if !r.Pinned() { + if !r.Pinned() || AvailableElsewhere(mbs, prefix, r.Key) { stillOkay = append(stillOkay, r.Key) } else { out <- &RemovedBlock{ @@ -70,6 +79,16 @@ func FilterPinned(pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid) []*c return stillOkay } +func AvailableElsewhere(mbs bs.MultiBlockstore, prefix string, c *cid.Cid) bool { + locations := mbs.Locate(c) + for _, loc := range locations { + if loc.Error == nil && loc.Prefix != prefix { + return true + } + } + return false +} + func ProcRmOutput(in <-chan interface{}, sout io.Writer, serr io.Writer) error { someFailed := false for res := range in { diff --git a/cmd/ipfs/ipfs.go b/cmd/ipfs/ipfs.go index f8c903346f4..58faa6e708a 100644 --- a/cmd/ipfs/ipfs.go +++ b/cmd/ipfs/ipfs.go @@ -108,4 +108,6 @@ var cmdDetailsMap = map[*cmds.Command]cmdDetails{ commands.ActiveReqsCmd: {cannotRunOnClient: true}, commands.RepoFsckCmd: {cannotRunOnDaemon: true}, commands.ConfigCmd.Subcommand("edit"): {cannotRunOnDaemon: true, doesNotUseRepo: true}, + commands.FilestoreEnable: {cannotRunOnDaemon: true}, + commands.FilestoreDisable: {cannotRunOnDaemon: true}, } diff --git a/commands/files/slicefile.go b/commands/files/slicefile.go index 8d18dcaa372..3282202f39e 100644 --- a/commands/files/slicefile.go +++ b/commands/files/slicefile.go @@ -23,6 +23,10 @@ func (f *SliceFile) IsDirectory() bool { return true } +func (f *SliceFile) NumFiles() int { + return len(f.files) +} + func (f *SliceFile) NextFile() (File, error) { if f.n >= len(f.files) { return nil, io.EOF diff --git a/core/builder.go b/core/builder.go index baef82ed06d..9a563cf2b08 100644 --- a/core/builder.go +++ b/core/builder.go @@ -16,6 +16,7 @@ import ( pin "github.com/ipfs/go-ipfs/pin" repo "github.com/ipfs/go-ipfs/repo" cfg "github.com/ipfs/go-ipfs/repo/config" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" context "context" retry "gx/ipfs/QmPF5kxTYFkzhaY5LmkExood7aTTZBHWQC6cjdDQBuGrjp/retry-datastore" @@ -26,6 +27,9 @@ import ( ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" dsync "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync" ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto" + + "github.com/ipfs/go-ipfs/filestore" + "github.com/ipfs/go-ipfs/filestore/support" ) type BuildCfg struct { @@ -167,7 +171,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { } var err error - bs := bstore.NewBlockstore(rds) + bs := bstore.NewBlockstoreWPrefix(rds, fsrepo.CacheMount) opts := bstore.DefaultCacheOpts() conf, err := n.Repo.Config() if err != nil { @@ -184,7 +188,14 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { return err } - n.Blockstore = bstore.NewGCBlockstore(cbs, bstore.NewGCLocker()) + mounts := []bstore.Mount{{fsrepo.CacheMount, cbs}} + + if n.Repo.DirectMount(fsrepo.FilestoreMount) != nil { + fs := bstore.NewBlockstoreWPrefix(n.Repo.Datastore(), fsrepo.FilestoreMount) + mounts = append(mounts, bstore.Mount{fsrepo.FilestoreMount, fs}) + } + + n.Blockstore = bstore.NewMultiBlockstore(mounts...) rcfg, err := n.Repo.Config() if err != nil { @@ -206,9 +217,13 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { n.Blocks = bserv.New(n.Blockstore, n.Exchange) n.DAG = dag.NewDAGService(n.Blocks) + if fs, ok := n.Repo.DirectMount(fsrepo.FilestoreMount).(*filestore.Datastore); ok { + n.DAG = filestore_support.NewDAGService(fs, n.DAG) + } internalDag := dag.NewDAGService(bserv.New(n.Blockstore, offline.Exchange(n.Blockstore))) n.Pinning, err = pin.LoadPinner(n.Repo.Datastore(), n.DAG, internalDag) + if err != nil { // TODO: we should move towards only running 'NewPinner' explicity on // node init instead of implicitly here as a result of the pinner keys diff --git a/core/commands/add.go b/core/commands/add.go index 52613ca4cf1..ab35321c546 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -1,13 +1,18 @@ package commands import ( + "errors" "fmt" "io" "github.com/ipfs/go-ipfs/core/coreunix" + "github.com/ipfs/go-ipfs/filestore" + "github.com/ipfs/go-ipfs/filestore/support" + "github.com/ipfs/go-ipfs/repo/fsrepo" "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" - blockservice "github.com/ipfs/go-ipfs/blockservice" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" + bserv "github.com/ipfs/go-ipfs/blockservice" cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" core "github.com/ipfs/go-ipfs/core" @@ -33,6 +38,7 @@ const ( chunkerOptionName = "chunker" pinOptionName = "pin" rawLeavesOptionName = "raw-leaves" + allowDupName = "allow-dup" ) var AddCmd = &cmds.Command{ @@ -80,8 +86,16 @@ You can now refer to the added file in a gateway, like so: cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."), cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true), cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"), + cmds.BoolOption(allowDupName, "Add even if blocks are in non-cache blockstore.").Default(false), }, PreRun: func(req cmds.Request) error { + wrap, _, _ := req.Option(wrapOptionName).Bool() + recursive, _, _ := req.Option(cmds.RecLong).Bool() + sliceFile, ok := req.Files().(*files.SliceFile) + if ok && !wrap && recursive && sliceFile.NumFiles() > 1 { + return fmt.Errorf("adding multiple directories without '-w' unsupported") + } + if quiet, _, _ := req.Option(quietOptionName).Bool(); quiet { return nil } @@ -138,6 +152,10 @@ You can now refer to the added file in a gateway, like so: chunker, _, _ := req.Option(chunkerOptionName).String() dopin, _, _ := req.Option(pinOptionName).Bool() rawblks, _, _ := req.Option(rawLeavesOptionName).Bool() + recursive, _, _ := req.Option(cmds.RecLong).Bool() + allowDup, _, _ := req.Option(allowDupName).Bool() + + nocopy, _ := req.Values()["no-copy"].(bool) if hash { nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{ @@ -152,18 +170,44 @@ You can now refer to the added file in a gateway, like so: n = nilnode } - dserv := n.DAG + exchange := n.Exchange local, _, _ := req.Option("local").Bool() if local { - offlineexch := offline.Exchange(n.Blockstore) - bserv := blockservice.New(n.Blockstore, offlineexch) - dserv = dag.NewDAGService(bserv) + exchange = offline.Exchange(n.Blockstore) } outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) - fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv) + var fileAdder *coreunix.Adder + useRoot := wrap || recursive + perFileLocker := filestore.NoOpLocker() + if nocopy { + fs, ok := n.Repo.DirectMount(fsrepo.FilestoreMount).(*filestore.Datastore) + if !ok { + res.SetError(errors.New("filestore not enabled"), cmds.ErrNormal) + return + } + blockstore := filestore_support.NewBlockstore(n.Blockstore, fs) + blockService := bserv.NewWriteThrough(blockstore, exchange) + dagService := dag.NewDAGService(blockService) + fileAdder, err = coreunix.NewAdder(req.Context(), n.Pinning, blockstore, dagService, useRoot) + fileAdder.FullName = true + perFileLocker = fs.AddLocker() + } else if allowDup { + // add directly to the first mount bypassing + // the Has() check of the multi-blockstore + blockstore := bs.NewGCBlockstore(n.Blockstore.FirstMount(), n.Blockstore) + blockService := bserv.NewWriteThrough(blockstore, exchange) + dagService := dag.NewDAGService(blockService) + fileAdder, err = coreunix.NewAdder(req.Context(), n.Pinning, blockstore, dagService, useRoot) + } else if exchange != n.Exchange { + blockService := bserv.New(n.Blockstore, exchange) + dagService := dag.NewDAGService(blockService) + fileAdder, err = coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dagService, useRoot) + } else { + fileAdder, err = coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, n.DAG, useRoot) + } if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -202,6 +246,8 @@ You can now refer to the added file in a gateway, like so: } else if err != nil { return err } + perFileLocker.Lock() + defer perFileLocker.Unlock() if err := fileAdder.AddFile(file); err != nil { return err } diff --git a/core/commands/block.go b/core/commands/block.go index 54b447d2ee1..0164e6f0a34 100644 --- a/core/commands/block.go +++ b/core/commands/block.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/ipfs/go-ipfs/blocks" + bs "github.com/ipfs/go-ipfs/blocks/blockstore" util "github.com/ipfs/go-ipfs/blocks/blockstore/util" cmds "github.com/ipfs/go-ipfs/commands" @@ -36,10 +37,11 @@ multihash. }, Subcommands: map[string]*cmds.Command{ - "stat": blockStatCmd, - "get": blockGetCmd, - "put": blockPutCmd, - "rm": blockRmCmd, + "stat": blockStatCmd, + "get": blockGetCmd, + "put": blockPutCmd, + "rm": blockRmCmd, + "locate": blockLocateCmd, }, } @@ -238,39 +240,104 @@ It takes a list of base58 encoded multihashs to remove. cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false), }, Run: func(req cmds.Request, res cmds.Response) { - n, err := req.InvocContext().GetNode() + blockRmRun(req, res, "") + }, + PostRun: func(req cmds.Request, res cmds.Response) { + if res.Error() != nil { + return + } + outChan, ok := res.Output().(<-chan interface{}) + if !ok { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + res.SetOutput(nil) + + err := util.ProcRmOutput(outChan, res.Stdout(), res.Stderr()) if err != nil { res.SetError(err, cmds.ErrNormal) - return } - hashes := req.Arguments() - force, _, _ := req.Option("force").Bool() - quiet, _, _ := req.Option("quiet").Bool() - cids := make([]*cid.Cid, 0, len(hashes)) - for _, hash := range hashes { - c, err := cid.Decode(hash) - if err != nil { - res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmds.ErrNormal) - return - } + }, + Type: util.RemovedBlock{}, +} - cids = append(cids, c) +func blockRmRun(req cmds.Request, res cmds.Response, prefix string) { + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + hashes := req.Arguments() + force, _, _ := req.Option("force").Bool() + quiet, _, _ := req.Option("quiet").Bool() + cids := make([]*cid.Cid, 0, len(hashes)) + for _, hash := range hashes { + c, err := cid.Decode(hash) + if err != nil { + res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmds.ErrNormal) + return } - outChan := make(chan interface{}) - err = util.RmBlocks(n.Blockstore, n.Pinning, outChan, cids, util.RmBlocksOpts{ - Quiet: quiet, - Force: force, - }) + cids = append(cids, c) + } + outChan := make(chan interface{}) + err = util.RmBlocks(n.Blockstore, n.Pinning, outChan, cids, util.RmBlocksOpts{ + Quiet: quiet, + Force: force, + Prefix: prefix, + }) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + res.SetOutput((<-chan interface{})(outChan)) +} + +type BlockLocateRes struct { + Key string + Res []bs.LocateInfo +} + +var blockLocateCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Locate an IPFS block.", + ShortDescription: ` +'ipfs block rm' is a plumbing command for locating which +sub-datastores block(s) are located in. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to check."), + }, + Options: []cmds.Option{ + cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmds.ErrNormal) return } + hashes := req.Arguments() + outChan := make(chan interface{}) res.SetOutput((<-chan interface{})(outChan)) + go func() { + defer close(outChan) + for _, hash := range hashes { + key, err := cid.Decode(hash) + if err != nil { + panic(err) // FIXME + } + ret := n.Blockstore.Locate(key) + outChan <- &BlockLocateRes{hash, ret} + } + }() + return }, PostRun: func(req cmds.Request, res cmds.Response) { if res.Error() != nil { return } + quiet, _, _ := req.Option("quiet").Bool() outChan, ok := res.Output().(<-chan interface{}) if !ok { res.SetError(u.ErrCast(), cmds.ErrNormal) @@ -278,10 +345,18 @@ It takes a list of base58 encoded multihashs to remove. } res.SetOutput(nil) - err := util.ProcRmOutput(outChan, res.Stdout(), res.Stderr()) - if err != nil { - res.SetError(err, cmds.ErrNormal) + for out := range outChan { + ret := out.(*BlockLocateRes) + for _, inf := range ret.Res { + if quiet && inf.Error == nil { + fmt.Fprintf(res.Stdout(), "%s %s\n", ret.Key, inf.Prefix) + } else if !quiet && inf.Error == nil { + fmt.Fprintf(res.Stdout(), "%s %s found\n", ret.Key, inf.Prefix) + } else if !quiet { + fmt.Fprintf(res.Stdout(), "%s %s error %s\n", ret.Key, inf.Prefix, inf.Error.Error()) + } + } } }, - Type: util.RemovedBlock{}, + Type: BlockLocateRes{}, } diff --git a/core/commands/filestore.go b/core/commands/filestore.go new file mode 100644 index 00000000000..ca18e6fac6f --- /dev/null +++ b/core/commands/filestore.go @@ -0,0 +1,994 @@ +package commands + +import ( + "bytes" + "errors" + "fmt" + "io" + //"io/ioutil" + "os" + "path/filepath" + "strings" + + //ds "github.com/ipfs/go-datastore" + //bs "github.com/ipfs/go-ipfs/blocks/blockstore" + cmds "github.com/ipfs/go-ipfs/commands" + cli "github.com/ipfs/go-ipfs/commands/cli" + files "github.com/ipfs/go-ipfs/commands/files" + "github.com/ipfs/go-ipfs/core" + "github.com/ipfs/go-ipfs/filestore" + fsutil "github.com/ipfs/go-ipfs/filestore/util" + "github.com/ipfs/go-ipfs/repo/fsrepo" + "gx/ipfs/QmRpAnJ1Mvd2wCtwoFevW8pbLTivUqmFxynptG6uvp1jzC/safepath" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + butil "github.com/ipfs/go-ipfs/blocks/blockstore/util" +) + +var FileStoreCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Interact with filestore objects.", + }, + Subcommands: map[string]*cmds.Command{ + "add": addFileStore, + "ls": lsFileStore, + "ls-files": lsFiles, + "verify": verifyFileStore, + "rm": rmFilestoreObjs, + "clean": cleanFileStore, + "dups": fsDups, + "mv": moveIntoFilestore, + "enable": FilestoreEnable, + "disable": FilestoreDisable, + }, +} + +var addFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Add files to the filestore.", + ShortDescription: ` +Add contents of to the filestore. Most of the options are the +same as for 'ipfs add'. +`}, + Arguments: []cmds.Argument{ + cmds.StringArg("path", true, true, "The path to a file to be added."), + }, + Options: addFileStoreOpts(), + PreRun: func(req cmds.Request) error { + serverSide, _, _ := req.Option("server-side").Bool() + logical, _, _ := req.Option("logical").Bool() + physical, _, _ := req.Option("physical").Bool() + if logical && physical { + return errors.New("both --logical and --physical can not be specified") + } + cwd := "" + var err error + if logical { + cwd, err = safepath.EnvWd() + } + if physical { + cwd, err = safepath.SystemWd() + } + if err != nil { + return err + } + if cwd != "" { + paths := req.Arguments() + for i, path := range paths { + abspath, err := safepath.AbsPath(cwd, path) + if err != nil { + return err + } + paths[i] = abspath + } + req.SetArguments(paths) + } + if !serverSide { + err := getFiles(req) + if err != nil { + return err + } + } + return AddCmd.PreRun(req) + }, + Run: func(req cmds.Request, res cmds.Response) { + node, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + config, _ := req.InvocContext().GetConfig() + serverSide, _, _ := req.Option("server-side").Bool() + if serverSide && !config.Filestore.APIServerSidePaths { + res.SetError(errors.New("server side paths not enabled"), cmds.ErrNormal) + return + } + if serverSide { + err := getFiles(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } else if !node.LocalMode() { + if !req.Files().IsDirectory() { + res.SetError(errors.New("expected directory object"), cmds.ErrNormal) + return + } + req.SetFiles(&fixPath{req.Arguments(), req.Files()}) + } + req.Values()["no-copy"] = true + AddCmd.Run(req, res) + }, + PostRun: AddCmd.PostRun, + Type: AddCmd.Type, +} + +func addFileStoreOpts() []cmds.Option { + var opts []cmds.Option + + foundPinOpt := false + for _, opt := range AddCmd.Options { + if opt.Names()[0] == pinOptionName { + opts = append(opts, cmds.BoolOption(pinOptionName, opt.Description()).Default(false)) + foundPinOpt = true + } else { + opts = append(opts, opt) + } + } + if !foundPinOpt { + panic("internal error: foundPinOpt is false") + } + + opts = append(opts, + cmds.BoolOption("server-side", "S", "Read file on server."), + cmds.BoolOption("logical", "l", "Create absolute path using PWD from environment."), + cmds.BoolOption("physical", "P", "Create absolute path using a system call."), + ) + return opts +} + +func getFiles(req cmds.Request) error { + inputs := req.Arguments() + for _, fn := range inputs { + if !filepath.IsAbs(fn) { + return fmt.Errorf("file path must be absolute: %s", fn) + } + } + _, fileArgs, err := cli.ParseArgs(req, inputs, nil, AddCmd.Arguments, nil) + if err != nil { + return err + } + file := files.NewSliceFile("", "", fileArgs) + req.SetFiles(file) + names := make([]string, len(fileArgs)) + for i, f := range fileArgs { + names[i] = f.FullPath() + } + req.SetArguments(names) + return nil +} + +type fixPath struct { + paths []string + orig files.File +} + +func (f *fixPath) IsDirectory() bool { return true } +func (f *fixPath) Read(res []byte) (int, error) { return 0, io.EOF } +func (f *fixPath) FileName() string { return f.orig.FileName() } +func (f *fixPath) FullPath() string { return f.orig.FullPath() } +func (f *fixPath) Close() error { return f.orig.Close() } + +func (f *fixPath) NextFile() (files.File, error) { + f0, _ := f.orig.NextFile() + if f0 == nil { + return nil, io.EOF + } + if len(f.paths) == 0 { + return nil, errors.New("len(req.Files()) < len(req.Arguments())") + } + path := f.paths[0] + f.paths = f.paths[1:] + if f0.IsDirectory() { + return nil, fmt.Errorf("online directory add not supported, try '-S': %s", path) + } else if _, ok := f0.(*files.MultipartFile); !ok { + return nil, fmt.Errorf("online adding of special files not supported, try '-S': %s", path) + } else { + f, err := os.Open(path) + if err != nil { + return nil, err + } + stat, err := f.Stat() + if err != nil { + return nil, err + } + return &dualFile{ + content: f0, + local: files.NewReaderFile(f0.FileName(), path, f, stat), + }, nil + } +} + +type dualFile struct { + content files.File + local files.StatFile + buf []byte +} + +func (f *dualFile) IsDirectory() bool { return false } +func (f *dualFile) NextFile() (files.File, error) { return nil, files.ErrNotDirectory } +func (f *dualFile) FileName() string { return f.local.FileName() } +func (f *dualFile) FullPath() string { return f.local.FullPath() } +func (f *dualFile) Stat() os.FileInfo { return f.local.Stat() } +func (f *dualFile) Size() (int64, error) { return f.local.Stat().Size(), nil } + +func (f *dualFile) Read(res []byte) (int, error) { + // First read the content send from the client + n, err1 := f.content.Read(res) + if err1 == io.ErrUnexpectedEOF { // avoid this special case + err1 = io.EOF + } + if err1 != nil && err1 != io.EOF { + return 0, err1 + } + res = res[:n] + + // Next try to read the same amount of data from the local file + if n == 0 && err1 == io.EOF { + // Make sure we try to read at least one byte in order + // to get an EOF + n = 1 + } + if cap(f.buf) < n { + f.buf = make([]byte, n) + } else { + f.buf = f.buf[:n] + } + n, err := io.ReadFull(f.local, f.buf) + if err == io.ErrUnexpectedEOF { // avoid this special case + err = io.EOF + } + if err != nil && err != io.EOF { + return 0, err + } + f.buf = f.buf[:n] + + // Now compare the results and return an error if the contents + // sent from the client differ from the contents of the file + if len(res) == 0 && err1 == io.EOF { + if len(f.buf) == 0 && err == io.EOF { + return 0, io.EOF + } else { + return 0, fmt.Errorf("%s: server side file is larger", f.FullPath()) + } + } + if !bytes.Equal(res, f.buf) { + return 0, fmt.Errorf("%s: %s: server side file contents differ", f.content.FullPath(), f.local.FullPath()) + } + return n, err1 +} + +func (f *dualFile) Close() error { + err := f.content.Close() + if err != nil { + return err + } + return f.local.Close() +} + +const listingCommonText = ` +If one or more is specified only list those specific objects, +otherwise list all objects. An can either be a filestore key, +or an absolute path. If the path ends in '/' than it is assumed to be +a directory and all paths with that directory are included. +` + +var lsFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List objects in filestore.", + ShortDescription: ` +List objects in the filestore. +` + listingCommonText + ` +If --all is specified then all matching blocks are listed, otherwise +only blocks representing a file root are listed. A file root is any +block that represents a complete file. + +The default output format normally is: + // +for most entries, if there are multiple files with the same content +then the first file will be as above and the others will be displayed +without the space between the and '/' to form a unique key that +can be used to reference that particular entry. + +If --format is "hash" than only the hash will be displayed. + +If --format is "key" than the full key will be displayed. + +If --format is "w/type" then additional information on the type of the +object is given, in the form of: + +where tree-type is one of: + ROOT: to indicate a root node that represents the whole file + leaf: to indicate a leaf node + other: to indicate some other type of node +and block-type is either blank or one of: + extrn: to indicate the data for the node is in a file + invld: to indicate the node is invalid due to the file changing + +If --format is "long" then the format is: + [] [ ]// +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("obj", false, true, "Hash(es), filename(s), or filestore keys to list."), + }, + Options: append(formatOpts, + cmds.BoolOption("all", "a", "List everything, not just file roots."), + ), + Run: func(req cmds.Request, res cmds.Response) { + _, fs, err := extractFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + all, _, _ := req.Option("all").Bool() + + formatFun, noObjInfo, err := procFormatOpts(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + ch, err := getListing(fs, req.Arguments(), all, noObjInfo) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + res.SetOutput(&chanWriter{ + ch: ch, + format: formatFun, + }) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +var formatOpts = []cmds.Option{ + cmds.BoolOption("quiet", "q", "Alias for --format=hash."), + cmds.BoolOption("full-key", "Display each entry using the full key when possible."), + cmds.StringOption("format", "f", "Format of listing, one of: hash key default w/type long").Default("default"), +} + +func procFormatOpts(req cmds.Request) (func(*fsutil.ListRes) (string, error), bool, error) { + format, _, _ := req.Option("format").String() + quiet, _, _ := req.Option("quiet").Bool() + fullKey, _, _ := req.Option("full-key").Bool() + + if quiet { + format = "hash" + } + + formatFun, err := fsutil.StrToFormatFun(format, fullKey) + if err != nil { + return nil, false, err + } + + return formatFun, format == "hash" || format == "key", nil +} + +func procListArgs(objs []string) ([]*filestore.DbKey, fsutil.ListFilter, error) { + keys := make([]*filestore.DbKey, 0) + paths := make([]string, 0) + for _, obj := range objs { + if filepath.IsAbs(obj) { + paths = append(paths, safepath.Clean(obj)) + } else { + key, err := filestore.ParseKey(obj) + if err != nil { + return nil, nil, err + } + keys = append(keys, key) + } + } + if len(keys) > 0 && len(paths) > 0 { + return nil, nil, errors.New("cannot specify both hashes and paths") + } + if len(keys) > 0 { + return keys, nil, nil + } else if len(paths) > 0 { + return nil, func(r *filestore.DataObj) bool { + return pathMatch(paths, r.FilePath) + }, nil + } else { + return nil, nil, nil + } +} + +func getListing(ds *filestore.Datastore, objs []string, all bool, keysOnly bool) (<-chan fsutil.ListRes, error) { + keys, listFilter, err := procListArgs(objs) + if err != nil { + return nil, err + } + + fs := ds.AsBasic() + + if len(keys) > 0 { + return fsutil.ListByKey(fs, keys) + } + + // Add filter filters if necessary + if !all { + if listFilter == nil { + listFilter = fsutil.ListFilterWholeFile + } else { + origFilter := listFilter + listFilter = func(r *filestore.DataObj) bool { + return fsutil.ListFilterWholeFile(r) && origFilter(r) + } + } + } + + return fsutil.List(fs, listFilter, keysOnly) +} + +func pathMatch(match_list []string, path string) bool { + for _, to_match := range match_list { + if to_match[len(to_match)-1] == filepath.Separator { + if strings.HasPrefix(path, to_match) { + return true + } + } else { + if to_match == path { + return true + } + } + } + return false +} + +var lsFiles = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List files in filestore.", + ShortDescription: ` +List files in the filestore. +` + listingCommonText + ` +If --quiet is specified only the file names are printed, otherwise the +fields are as follows: + +`, + }, + Arguments: lsFileStore.Arguments, + Options: []cmds.Option{ + cmds.BoolOption("quiet", "q", "Write just filenames."), + }, + Run: func(req cmds.Request, res cmds.Response) { + _, fs, err := extractFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + quiet, _, err := req.Option("quiet").Bool() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + ch, err := getListing(fs, req.Arguments(), false, false) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + if quiet { + res.SetOutput(&chanWriter{ch: ch, format: formatFileName}) + } else { + res.SetOutput(&chanWriter{ch: ch, format: formatByFile}) + } + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +type chanWriter struct { + ch <-chan fsutil.ListRes + buf string + offset int + checksFailed bool + ignoreFailed bool + errs []string + format func(*fsutil.ListRes) (string, error) +} + +func (w *chanWriter) Read(p []byte) (int, error) { + if w.offset >= len(w.buf) { + w.offset = 0 + res, more := <-w.ch + + if !more { + if w.checksFailed { + w.errs = append(w.errs, "some checks failed") + } + if len(w.errs) == 0 { + return 0, io.EOF + } else { + return 0, errors.New(strings.Join(w.errs, ". ")) + } + } + + if !w.ignoreFailed && fsutil.AnError(res.Status) { + w.checksFailed = true + } + + line, err := w.format(&res) + w.buf = line + if err != nil { + w.errs = append(w.errs, fmt.Sprintf("%s: %s", res.MHash(), err.Error())) + } + } + sz := copy(p, w.buf[w.offset:]) + w.offset += sz + return sz, nil +} + +func formatDefault(res *fsutil.ListRes) (string, error) { + return res.FormatDefault(false), nil +} + +func formatHash(res *fsutil.ListRes) (string, error) { + return res.FormatHashOnly(), nil +} + +func formatPorcelain(res *fsutil.ListRes) (string, error) { + if res.Key.Hash == "" { + return "", nil + } + if res.DataObj == nil { + return fmt.Sprintf("%s\t%s\t%s\t%s\t%s\n", "block", res.StatusStr(), res.MHash(), "", ""), nil + } + pos := strings.IndexAny(res.FilePath, "\t\r\n") + if pos == -1 { + return fmt.Sprintf("%s\t%s\t%s\t%s\t%d\n", res.What(), res.StatusStr(), res.MHash(), res.FilePath, res.Offset), nil + } else { + str := fmt.Sprintf("%s\t%s\t%s\t%s\t%d\n", res.What(), res.StatusStr(), res.MHash(), "", res.Offset) + err := errors.New("not displaying filename with tab or newline character") + return str, err + } +} + +func formatFileName(res *fsutil.ListRes) (string, error) { + return fmt.Sprintf("%s\n", res.FilePath), nil +} + +func formatByFile(res *fsutil.ListRes) (string, error) { + return fmt.Sprintf("%s %s %d\n", res.FilePath, res.MHash(), res.Size), nil +} + +var verifyFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Verify objects in filestore.", + ShortDescription: ` +Verify nodes in the filestore. If no hashes are specified then +verify everything in the filestore. + +The normal output is: + /// +where , are the same as in the 'ls' command +and is one of: + + ok: the original data can be reconstructed + complete: all the blocks in the tree exists but no attempt was + made to reconstruct the original data + + problem: some of the blocks of the tree could not be read + incomplete: some of the blocks of the tree are missing + + changed: the contents of the backing file have changed + no-file: the backing file could not be found + error: the backing file was found but could not be read + + touched: the modtimes differ but the contents where not rechecked + + ERROR: the block could not be read due to an internal error + + found: the child of another node was found outside the filestore + missing: the child of another node does not exist + : the child of another node node exists but no attempt was + made to verify it + + appended: the node is still valid but the original file was appended + + orphan: the node is a child of another node that was not found in + the filestore + +If any checks failed than a non-zero exit status will be returned. + +If --basic is specified linearly scan the leaf nodes to verify that they +are still valid. Otherwise attempt to reconstruct the contents of all +nodes and check for orphan nodes if applicable. + +Otherwise, the nodes are recursively visited from the root node. If +--skip-orphans is not specified than the results are cached in memory in +order to detect the orphans. The cache is also used to avoid visiting +the same node more than once. Cached results are printed without any +object info. + +The --level option specifies how thorough the checks should be. The +current meaning of the levels are: + 7-9: always check the contents + 6: check the contents based on the setting of Filestore.Verify + 4-5: check the contents if the modification time differs + 2-3: report if the modification time differs + 0-1: only check for the existence of blocks without verifying the + contents of leaf nodes + +The --verbose option specifies what to output. The current values are: + 0-1: show top-level nodes when status is not 'ok', 'complete' or '' + 2: in addition, show all nodes specified on command line + 3-4: in addition, show all top-level nodes + 5-6: in addition, show problem children + 7-9: in addition, show all children + +If --porcelain is used used an alternative output is used that will not +change between releases. The output is: + \t\t\t\t +where is either "root" for a file root or something else +otherwise and \t is a literal literal tab character. is the +same as normal except that is spelled out as "unchecked". In +addition to the modified output a non-zero exit status will only be +returned on an error condition and not just because of failed checks. +In the event that contains a tab or newline character the +filename will not be displayed (and a non-zero exit status will be +returned) to avoid special cases when parsing the output. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("obj", false, true, "Hash(es), filename(s), or filestore keys to verify."), + }, + Options: append(formatOpts, + cmds.BoolOption("basic", "Perform a basic scan of leaf nodes only."), + cmds.IntOption("level", "l", "0-9, Verification level.").Default(6), + cmds.IntOption("verbose", "v", "0-9 Verbose level.").Default(6), + cmds.BoolOption("porcelain", "Porcelain output."), + cmds.BoolOption("skip-orphans", "Skip check for orphans."), + cmds.StringOption("incomplete-when", "Internal option."), + cmds.BoolOption("post-orphan", "Internal option: Report would-be orphans."), + ), + Run: func(req cmds.Request, res cmds.Response) { + node, fs, err := extractFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + args := req.Arguments() + keys, filter, err := procListArgs(args) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + formatFun, noObjInfo, err := procFormatOpts(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + basic, _, _ := req.Option("basic").Bool() + porcelain, _, _ := req.Option("porcelain").Bool() + + params := fsutil.VerifyParams{Filter: filter, NoObjInfo: noObjInfo} + params.Level, _, _ = req.Option("level").Int() + params.Verbose, _, _ = req.Option("verbose").Int() + params.SkipOrphans, _, _ = req.Option("skip-orphans").Bool() + params.IncompleteWhen = getIncompleteWhenOpt(req) + params.PostOrphan, _, _ = req.Option("post-orphan").Bool() + + var ch <-chan fsutil.ListRes + if basic && len(keys) == 0 { + ch, err = fsutil.VerifyBasic(fs.AsBasic(), ¶ms) + } else if basic { + ch, err = fsutil.VerifyKeys(keys, node, fs.AsBasic(), ¶ms) + } else if len(keys) == 0 { + snapshot, err0 := fs.GetSnapshot() + if err0 != nil { + res.SetError(err0, cmds.ErrNormal) + return + } + ch, err = fsutil.VerifyFull(node, snapshot, ¶ms) + } else { + ch, err = fsutil.VerifyKeysFull(keys, node, fs.AsBasic(), ¶ms) + } + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + if porcelain { + res.SetOutput(&chanWriter{ch: ch, format: formatPorcelain, ignoreFailed: true}) + } else { + res.SetOutput(&chanWriter{ch: ch, format: formatFun}) + } + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +func getIncompleteWhenOpt(req cmds.Request) []string { + str, _, _ := req.Option("incomplete-when").String() + if str == "" { + return nil + } else { + return strings.Split(str, ",") + } +} + +var cleanFileStore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove invalid or orphan nodes from the filestore.", + ShortDescription: ` +Removes invalid or orphan nodes from the filestore as specified by +. is the status of a node as reported by "verify", it +can be any of "changed", "no-file", "error", "incomplete", +"orphan", "invalid" or "full". "invalid" is an alias for "changed" +and "no-file" and "full" is an alias for "invalid" "incomplete" and +"orphan" (basically remove everything but "error"). + +If incomplete is specified in combination with "changed", "no-file", or +"error" than any nodes that will become incomplete, after the invalid leafs +are removed, are also removed. Similarly if "orphan" is specified in +combination with "incomplete" any would be orphans are also removed. + +If the command is run with the daemon is running the check is done on a +snapshot of the filestore when it is in a consistent state. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("what", true, true, "any of: changed no-file error incomplete orphan invalid full"), + }, + Options: []cmds.Option{ + cmds.BoolOption("quiet", "q", "Produce less output."), + cmds.IntOption("level", "l", "0-9, Verification level.").Default(6), + }, + Run: func(req cmds.Request, res cmds.Response) { + node, fs, err := extractFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + quiet, _, err := req.Option("quiet").Bool() + level, _, _ := req.Option("level").Int() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + rdr, err := fsutil.Clean(req, node, fs, quiet, level, req.Arguments()...) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + res.SetOutput(rdr) + //res.SetOutput(&chanWriter{ch, "", 0, false}) + return + }, + //Marshalers: cmds.MarshalerMap{ + // cmds.Text: func(res cmds.Response) (io.Reader, error) { + // return res.(io.Reader), nil + // }, + //}, +} + +var rmFilestoreObjs = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Remove entries from the filestore.", + ShortDescription: ` +Remove matching entries from the filestore. Entries can be specified +using any of the following formats: + + / + /// + +To prevent accidentally removing a block that is part of an unrelated +file, only roots will be removed unless either "--allow-non-roots" is +also specified or a key with a is provided. + +To remove all blocks associated with a file use "-r" to remove the +children, in addition to the root. The "-r" option is safe to use +even if blocks are shared between files, as it will only remove +children that have the same backing file. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("key", true, true, "Objects to remove."), + }, + Options: []cmds.Option{ + cmds.BoolOption("ignore", "Ignore nonexistent blocks."), + cmds.BoolOption("quiet", "q", "Write minimal output."), + cmds.BoolOption("recursive", "r", "Delete children."), + cmds.BoolOption("allow-non-roots", "Allow removal of non-root nodes when only the hash is specified."), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, fs, err := extractFilestore(req) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + args := req.Arguments() + + ss, err := fs.GetSnapshot() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + r := fsutil.NewFilestoreRemover(ss) + quiet, _, _ := req.Option("quiet").Bool() + r.ReportFound = !quiet + ignore, _, _ := req.Option("ignore").Bool() + r.ReportNotFound = !ignore + r.Recursive, _, _ = req.Option("recursive").Bool() + r.AllowNonRoots, _, _ = req.Option("allow-non-roots").Bool() + out := make(chan interface{}, 16) + go func() { + defer close(out) + for _, hash := range args { + k, err := filestore.ParseKey(hash) + if err != nil { + out <- &butil.RemovedBlock{Hash: hash, Error: "invalid filestore key"} + continue + } + r.DeleteAll(k, out) + } + out2 := r.Finish(n.Blockstore, n.Pinning) + for res := range out2 { + out <- res + } + }() + res.SetOutput((<-chan interface{})(out)) + }, + PostRun: blockRmCmd.PostRun, + Type: blockRmCmd.Type, +} + +func extractFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Datastore, error) { + node, err := req.InvocContext().GetNode() + if err != nil { + return nil, nil, err + } + fs, ok := node.Repo.DirectMount(fsrepo.FilestoreMount).(*filestore.Datastore) + if !ok { + err := errors.New("filestore not enabled") + return nil, nil, err + } + return node, fs, nil +} + +var fsDups = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "List duplicate blocks stored outside filestore.", + }, + Arguments: []cmds.Argument{ + cmds.StringArg("what", false, true, "any of: pinned unpinned"), + }, + Run: func(req cmds.Request, res cmds.Response) { + node, fs, err := extractFilestore(req) + if err != nil { + return + } + r, w := io.Pipe() + go func() { + err := fsutil.Dups(w, fs.AsBasic(), node.Blockstore, node.Pinning, req.Arguments()...) + if err != nil { + w.CloseWithError(err) + } else { + w.Close() + } + }() + res.SetOutput(r) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +var moveIntoFilestore = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Move a Node representing file into the filestore.", + ShortDescription: ` +Move a node representing a file into the filestore. For now the old +copy is not removed. Use "filestore rm-dups" to remove the old copy. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("hash", true, false, "Multi-hash to move."), + cmds.StringArg("file", false, false, "File to store node's content in."), + }, + Options: []cmds.Option{}, + Run: func(req cmds.Request, res cmds.Response) { + node, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + local := node.LocalMode() + args := req.Arguments() + if len(args) < 1 { + res.SetError(errors.New("must specify hash"), cmds.ErrNormal) + return + } + if len(args) > 2 { + res.SetError(errors.New("too many arguments"), cmds.ErrNormal) + return + } + mhash := args[0] + k, err := cid.Decode(mhash) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + path := "" + if len(args) == 2 { + path = args[1] + } else { + path = mhash + } + if local { + path, err = filepath.Abs(path) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + } + rdr, wtr := io.Pipe() + go func() { + err := fsutil.ConvertToFile(node, k, path) + if err != nil { + wtr.CloseWithError(err) + return + } + wtr.Close() + }() + res.SetOutput(rdr) + return + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + return res.(io.Reader), nil + }, + }, +} + +var FilestoreEnable = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Enable the filestore.", + ShortDescription: ` +Enable the filestore. A noop if the filestore is already enabled. +`, + }, + Run: func(req cmds.Request, res cmds.Response) { + rootDir := req.InvocContext().ConfigRoot + err := fsrepo.InitFilestore(rootDir) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + }, +} + +var FilestoreDisable = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Disable an empty filestore.", + ShortDescription: ` +Disable the filestore if it is empty. A noop if the filestore does +not exist. An error if the filestore is not empty. +`, + }, + Run: func(req cmds.Request, res cmds.Response) { + res.SetError(errors.New("unimplemented"), cmds.ErrNormal) + }, +} diff --git a/core/commands/root.go b/core/commands/root.go index 2238938ce9a..33adb2dd4df 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -120,6 +120,7 @@ var rootSubcommands = map[string]*cmds.Command{ "update": ExternalBinary(), "version": VersionCmd, "bitswap": BitswapCmd, + "filestore": FileStoreCmd, } // RootRO is the readonly version of Root diff --git a/core/core.go b/core/core.go index 231b91ac6b0..34bed4bccea 100644 --- a/core/core.go +++ b/core/core.go @@ -95,11 +95,11 @@ type IpfsNode struct { PrivateKey ic.PrivKey // the local node's private Key // Services - Peerstore pstore.Peerstore // storage for other Peer instances - Blockstore bstore.GCBlockstore // the block store (lower level) - Blocks bserv.BlockService // the block service, get/add blocks. - DAG merkledag.DAGService // the merkle dag service, get/add objects. - Resolver *path.Resolver // the path resolution system + Peerstore pstore.Peerstore // storage for other Peer instances + Blockstore bstore.MultiBlockstore // the block store (lower level) + Blocks bserv.BlockService // the block service, get/add blocks. + DAG merkledag.DAGService // the merkle dag service, get/add objects. + Resolver *path.Resolver // the path resolution system Reporter metrics.Reporter Discovery discovery.Service FilesRoot *mfs.Root @@ -120,7 +120,7 @@ type IpfsNode struct { proc goprocess.Process ctx context.Context - mode mode + mode mode localModeSet bool } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 5e224808db6..ae9d80fee58 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -2,6 +2,7 @@ package coreunix import ( "context" + "errors" "fmt" "io" "io/ioutil" @@ -67,14 +68,8 @@ type AddedObject struct { Bytes int64 `json:",omitempty"` } -func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) { - mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil) - if err != nil { - return nil, err - } - - return &Adder{ - mr: mr, +func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService, useRoot bool) (*Adder, error) { + adder := &Adder{ ctx: ctx, pinning: p, blockstore: bs, @@ -85,8 +80,17 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag. Trickle: false, Wrap: false, Chunker: "", - }, nil + } + + if useRoot { + mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil) + if err != nil { + return nil, err + } + adder.mr = mr + } + return adder, nil } // Internal structure for holding the switches passed to the `add` call @@ -104,6 +108,7 @@ type Adder struct { Silent bool Wrap bool Chunker string + FullName bool root node.Node mr *mfs.Root unlocker bs.Unlocker @@ -134,6 +139,10 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) { } func (adder *Adder) RootNode() (node.Node, error) { + if adder.mr == nil { + return nil, nil + } + // for memoizing if adder.root != nil { return adder.root, nil @@ -159,6 +168,10 @@ func (adder *Adder) RootNode() (node.Node, error) { } func (adder *Adder) PinRoot() error { + if adder.mr == nil { + return nil + } + root, err := adder.RootNode() if err != nil { return err @@ -185,6 +198,13 @@ func (adder *Adder) PinRoot() error { } func (adder *Adder) Finalize() (node.Node, error) { + if adder.mr == nil && adder.Pin { + err := adder.pinning.Flush() + return nil, err + } else if adder.mr == nil { + return nil, nil + } + root := adder.mr.GetValue() // cant just call adder.RootNode() here as we need the name for printing @@ -256,7 +276,7 @@ func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error { func Add(n *core.IpfsNode, r io.Reader) (string, error) { defer n.Blockstore.PinLock().Unlock() - fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) + fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG, true) if err != nil { return "", err } @@ -284,7 +304,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { } defer f.Close() - fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) + fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG, true) if err != nil { return "", err } @@ -308,7 +328,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { // the directory, and and error if any. func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) { file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil) - fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) + fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG, true) if err != nil { return "", nil, err } @@ -330,25 +350,38 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.No return gopath.Join(c.String(), filename), dagnode, nil } -func (adder *Adder) addNode(node node.Node, path string) error { - // patch it into the root - if path == "" { - path = node.Cid().String() - } +func (adder *Adder) pinOrAddNode(node node.Node, file files.File) error { + path := file.FileName() + + if adder.Pin && adder.mr == nil { + + adder.pinning.PinWithMode(node.Cid(), pin.Recursive) + + } else if adder.mr != nil { + + // patch it into the root + if path == "" { + path = node.Cid().String() + } + + dir := gopath.Dir(path) + if dir != "." { + if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil { + return err + } + } - dir := gopath.Dir(path) - if dir != "." { - if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil { + if err := mfs.PutNode(adder.mr, path, node); err != nil { return err } - } - if err := mfs.PutNode(adder.mr, path, node); err != nil { - return err } - if !adder.Silent { - return outputDagnode(adder.Out, path, node) + if adder.FullName { + return outputDagnode(adder.Out, file.FullPath(), node) + } else { + return outputDagnode(adder.Out, file.FileName(), node) + } } return nil } @@ -390,7 +423,7 @@ func (adder *Adder) addFile(file files.File) error { return err } - return adder.addNode(dagnode, s.FileName()) + return adder.pinOrAddNode(dagnode, s) } // case for regular file @@ -412,10 +445,14 @@ func (adder *Adder) addFile(file files.File) error { } // patch it into the root - return adder.addNode(dagnode, file.FileName()) + return adder.pinOrAddNode(dagnode, file) } func (adder *Adder) addDir(dir files.File) error { + if adder.mr == nil { + return errors.New("cannot add directories without mfs root") + } + log.Infof("adding directory: %s", dir.FileName()) err := mfs.Mkdir(adder.mr, dir.FileName(), true, false) diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 0a02c137051..281df1c5d9f 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -61,7 +61,7 @@ func TestAddGCLive(t *testing.T) { errs := make(chan error) out := make(chan interface{}) - adder, err := NewAdder(context.Background(), node.Pinning, node.Blockstore, node.DAG) + adder, err := NewAdder(context.Background(), node.Pinning, node.Blockstore, node.DAG, true) if err != nil { t.Fatal(err) } @@ -186,7 +186,7 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) { bs := &testBlockstore{GCBlockstore: node.Blockstore, expectedPath: "/tmp/foo.txt", t: t} bserv := blockservice.New(bs, node.Exchange) dserv := dag.NewDAGService(bserv) - adder, err := NewAdder(context.Background(), node.Pinning, bs, dserv) + adder, err := NewAdder(context.Background(), node.Pinning, bs, dserv, false) if err != nil { t.Fatal(err) } diff --git a/filestore/README.md b/filestore/README.md new file mode 100644 index 00000000000..42e3be9c3ce --- /dev/null +++ b/filestore/README.md @@ -0,0 +1,249 @@ +# Notes on the Filestore + +The filestore is a work-in-progress datastore that stores the unixfs +data component of blocks in files on the filesystem instead of in the +block itself. The main use of the datastore is to add content to IPFS +without duplicating the content in the IPFS datastore. + +The filestore is developed on Debian (GNU/Linux). It has has limited +testing on Windows and should work on MacOS X and other Unix like +systems. + +Before the filestore can be used it must be enabled with +``` + ipfs filestore enable +``` + +## Adding Files + +To add a file to IPFS without copying, use `filestore add -P` or to add a +directory use `filestore add -P -r`. (Throughout this document all +command are assumed to start with `ipfs` so `filestore add` really +mains `ipfs filestore add`). For example to add the file `hello.txt` +use: +``` + ipfs filestore add -P hello.txt +``` + +Paths stored in the filestore must be absolute. You can either +provide an absolute path or use one of `-P` (`--physical`) or `-l` +(`--logical`) to create one. The `-P` (or `--physical`) means to make +an absolute path from the physical working directory without any +symbolic links in it; the `-l` (or `--logical`) means to use the `PWD` +env. variable if possible. + +When adding a file with the daemon online the same file must be +accessible via the path provided by both the client and the server. +Without extra options it is currently not possible to add directories +with the daemon online. + +By default, the contents of the file are always verified by +recomputing the hash. The setting `Filestore.Verify` can be used to +change this to never recompute the hash (not recommended) or to only +recompute the hash when the modification-time has changed. + +Adding files to the filestore will generally be faster than adding +blocks normally as less data is copied around. Retrieving blocks from +the filestore takes about the same time when the hash is not +recomputed, when it is, retrieval is slower. + +## Adding all files in a directory + +FIXME: This section (and the add-dir script) need to be updated to +reflect the new semantics. + +Adding all files in a directory using `-r` is limited. For one thing, +it can normally only be done with the daemon offline. In addition it is +not a resumable operation. A better way is to use the "add-dir" script +found in the `examples/` directory. It usage is: +``` + add-dir [--scan] DIR [CACHE] +``` +In it's most basic usage it will work like `filestore add -r` but will +add files individually rather than as a directory. If the `--scan` +option is used the script will scan the filestore for any files +already added and only add new files or those that have changed. When +the `--scan` option is used to keep a directory in sync, duplicate +files will always be readded. In addition, if two files have +overlapping content it is not guaranteed to find all changes. To +avoid these problems a cache file can also be specified. + +If a cache file is specified, then, information about the files will +be stored in the file `CACHE` in order to keep the directory contents +in sync with what is in the filestore. The cache files is written out +as files are added so that if the script is aborted it will pick up +from where it left off the next time it is run. + +If the cache file does not exist and `--scan` is specified than the +cache will be initialized with what is in the filestore. + +A good use of the add-dir script is to add it to crontab to rerun the +script periodically. + +The add-dir does not perform any maintenance to remove blocks that +have become invalid so it would be a good idea to run something like +`ipfs filestore clean full` periodically. See the maintenance section +later in this document for more details. + +The `add-dir` script if fairly simple way to keep a directly in sync. +A more sophisticated application could use i-notify or a similar +interface to re-add files as they are changed. + +## Server side adds + +When adding a file when the daemon is online. The client sends both +the file contents and path to the server, and the server will then +verify that the same content is available via the specified path by +reading the file again on the server side. To avoid this extra +overhead and allow directories to be added when the daemon is +online server side paths can be used. + +To use this feature you must first enable API.ServerSideAdds using: +``` + ipfs config Filestore.APIServerSidePaths --bool true +``` +*This option should be used with care since it will allow anyone with +access to the API Server access to any files that the daemon has +permission to read.* For security reasons it is probably best to only +enable this on a single user system and to make sure the API server is +configured to the default value of only binding to the localhost +(`127.0.0.1`). + +With the `Filestore.APIServerSidePaths` option enabled you can add +files using `filestore add -S`. For example, to add the file +`hello.txt` in the current directory use: +``` + ipfs filestore add -S -P hello.txt +``` +## About filestore entries + +Each entry in the filestore is uniquely refereed to by combining the +(1) the hash of the block, (2) the path to the file, and (3) the +offset within the file, using the following syntax: +``` + /// +``` +for example: +``` + QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH//somedir/hello.txt//0 +``` + +In the case that there is only one entry for a hash the entry is +stored using just the hash. If there is more than one entry for a +hash (for example if adding two files with identical content) than one +entry will be stored using just the hash and the others will be stored +using the full key. If the backing file changes or becomes +inaccessible for the default entry (the one with just the hash) the +other entries are tried until a valid entry is found. Once a valid +entry is found that entry will become the default. + +When listing the contents of the filestore entries that are stored +using just the hash are displayed as +``` + /// +``` +with a space between the amd . + +It is always possible to refer to a specific entry in the filestore +using the full key regardless to how it is stored. + +## Listing and verifying blocks + +To list the contents of the filestore use the command `filestore ls`, +or `filestore ls-files`. See `--help` for additional information. + +To verify the contents of the filestore use `filestore verify`. +Again see `--help` for additional info. + +## Maintenance + +Invalid blocks should be cleared out from time to time. An invalid +block is a block where the data from the backing file is no longer +available. Operations that only depend on the DAG object metadata +will continue to function (such as `refs` and `repo gc`) but any +attempt to retrieve the block will fail. + +Currently no regular maintenance is done and it is unclear if this is +a desirable thing as I image the filestore will primary be used in +conjunction will some higher level tools that will automatically +manage the filestore. + +Before performing maintenance any invalid pinned blocks need to be +manually unpinned. The maintenance commands will skip pinned blocks. + +Maintenance commands are safe to run with the daemon running; however, +if other filestore modification operations are running in parallel, +the maintaince command may not be complete. Most maintenance commands +will operate on a snapshot of the database when it was last in a +consistent state. + +## Removing Invalid blocks + +The `filestore clean` command will remove invalid blocks as reported +by `filestore verify`. You must specify what type of invalid blocks to +remove. This command should be used with some care to avoid removing +more than is intended. For help with the command use +`filestore clean --help` + +Removing `changed` and `no-file` blocks (as reported by `filestore verify` +is generally a safe thing to do. When removing `no-file` blocks there +is a slight risk of removing blocks to files that might reappear, for +example, if a filesystem containing the file for the block is not +mounted. + +Removing `error` blocks runs the risk of removing blocks to files that +are not available due to transient or easily correctable errors (such as +permission problems). + +Removing `incomplete` blocks is generally safe as the interior node is +basically useless without the children. However, there is nothing +wrong with the block itself, so if the missing children are still +available elsewhere removing `incomplete` blocks is immature and might +lead to the lose of data. + +Removing `orphan` blocks, like `incomplete` blocks, runs the risk of +data lose if the root node is found elsewhere. Also, unlike +`incomplete` blocks `orphan` blocks may still be useful and only take +up a small amount of space. + +## Pinning and removing blocks manually. + +Filestore blocks are never garage collected and hence filestore blocks +are not pinned by default when added. If you add a directory it will +also not be pinned (as that will indirectly pin filestore objects) and +hence the directory object might be garbage collected as it is not +stored in the filestore. + +To manually remove entries in the filestore use `filestore rm`. + +## Duplicate blocks. + +If a block has already been added to the datastore, adding it again +with `filestore add` will add the block to the filestore but the now +duplicate block will still exists in the normal datastore. If the +block is not pinned it will be removed from the normal datastore when +garbage collected. If the block is pinned it will exist in both +locations. Removing the duplicate may not always be the most +desirable thing to do as filestore blocks are less stable. + +The command "filestore dups" will list duplicate blocks. "block rm" +can then be used to remove the blocks. It is okay to remove a +duplicate pinned block as long as at least one copy is still around. + +Once a file is in the filestore it will not be added to the normal +datastore, the option "--allow-dup" will override this behavior and +add the file anyway. This is useful for testing and to make a more +stable copy of an important peace of data. + +To determine the location of a block use "block locate". + +## Controlling when blocks are verified. + +The config variable `Filestore.Verify` can be used to customize when +blocks from the filestore are verified. The default value `Always` +will always verify blocks. A value of `IfChanged. will verify a +block if the modification time of the backing file has changed. This +value works well in most cases, but can miss some changes, espacally +if the filesystem only tracks file modification times with a +resolution of one second (HFS+, used by OS X) or less (FAT32). A +value of `Never`, never checks blocks. diff --git a/filestore/dataobj.go b/filestore/dataobj.go new file mode 100644 index 00000000000..3d49a005dd3 --- /dev/null +++ b/filestore/dataobj.go @@ -0,0 +1,186 @@ +package filestore + +import ( + "fmt" + pb "github.com/ipfs/go-ipfs/filestore/pb" + "math" + "time" +) + +const ( + // If NoBlockData is true the Data is missing the Block data + // as that is provided by the underlying file + NoBlockData = 1 + // If WholeFile is true the Data object represents a complete + // file and Size is the size of the file + WholeFile = 2 + // If the node represents an a file but is not a leaf + // If WholeFile is also true than it is the file's root node + Internal = 4 + // If the block was determined to no longer be valid + Invalid = 8 +) + +type DataObj struct { + Flags uint64 + // The path to the file that holds the data for the object, an + // empty string if there is no underlying file + FilePath string + Offset uint64 + Size uint64 + ModTime float64 + Data []byte +} + +func (d *DataObj) NoBlockData() bool { return d.Flags&NoBlockData != 0 } +func (d *DataObj) HaveBlockData() bool { return !d.NoBlockData() } + +func (d *DataObj) WholeFile() bool { return d.Flags&WholeFile != 0 } + +func (d *DataObj) Internal() bool { return d.Flags&Internal != 0 } + +func (d *DataObj) Invalid() bool { return d.Flags&Invalid != 0 } + +func (d *DataObj) SetInvalid(val bool) { + if val { + d.Flags |= Invalid + } else { + d.Flags &^= Invalid + } +} + +func FromTime(t time.Time) float64 { + res := float64(t.Unix()) + if res > 0 { + res += float64(t.Nanosecond()) / 1000000000.0 + } + return res +} + +func ToTime(t float64) time.Time { + sec, frac := math.Modf(t) + return time.Unix(int64(sec), int64(frac*1000000000.0)) +} + +func (d *DataObj) StripData() DataObj { + return DataObj{ + d.Flags, d.FilePath, d.Offset, d.Size, d.ModTime, nil, + } +} + +func (d *DataObj) KeyStr(key Key, asKey bool) string { + if key.FilePath == "" { + res := key.Format() + if asKey { + res += "/" + } else { + res += " /" + } + res += d.FilePath + res += "//" + res += fmt.Sprintf("%d", d.Offset) + return res + } else { + return key.Format() + } +} + +func (d *DataObj) TypeStr() string { + str := ""; + if d.WholeFile() { + str += "ROOT " + } else if d.Internal() { + str += "other"; + } else { + str += "leaf "; + } + if d.Invalid() && d.NoBlockData() { + str += " invld"; + } else if d.NoBlockData() { + str += " extrn"; + } else { + str += " "; + } + return str +} + +func (d *DataObj) DateStr() string { + if d.NoBlockData() { + return ToTime(d.ModTime).Format("2006-01-02T15:04:05.000Z07:00") + } else { + return "" + } +} + +func (d *DataObj) Format() string { + offset := fmt.Sprintf("%d", d.Offset) + if d.WholeFile() { + offset = "-" + } + date := ToTime(d.ModTime).Format("2006-01-02T15:04:05.000Z07:00") + if d.Invalid() && d.NoBlockData() { + return fmt.Sprintf("invld %s %s %d %s", d.FilePath, offset, d.Size, date) + } else if d.NoBlockData() { + return fmt.Sprintf("leaf %s %s %d %s", d.FilePath, offset, d.Size, date) + } else if d.Internal() && d.WholeFile() { + return fmt.Sprintf("root %s %s %d", d.FilePath, offset, d.Size) + } else { + return fmt.Sprintf("other %s %s %d", d.FilePath, offset, d.Size) + } +} + +func (d *DataObj) Marshal() ([]byte, error) { + pd := new(pb.DataObj) + + pd.Flags = &d.Flags + + if d.FilePath != "" { + pd.FilePath = &d.FilePath + } + if d.Offset != 0 { + pd.Offset = &d.Offset + } + if d.Size != 0 { + pd.Size_ = &d.Size + } + if d.Data != nil { + pd.Data = d.Data + } + + if d.ModTime != 0.0 { + pd.Modtime = &d.ModTime + } + + return pd.Marshal() +} + +func (d *DataObj) Unmarshal(data []byte) error { + pd := new(pb.DataObj) + err := pd.Unmarshal(data) + if err != nil { + panic(err) + } + + if pd.Flags != nil { + d.Flags = *pd.Flags + } + + if pd.FilePath != nil { + d.FilePath = *pd.FilePath + } + if pd.Offset != nil { + d.Offset = *pd.Offset + } + if pd.Size_ != nil { + d.Size = *pd.Size_ + } + if pd.Data != nil { + d.Data = pd.Data + } + + if pd.Modtime != nil { + d.ModTime = *pd.Modtime + } + + return nil +} diff --git a/filestore/datastore.go b/filestore/datastore.go new file mode 100644 index 00000000000..c05b1a7d9dd --- /dev/null +++ b/filestore/datastore.go @@ -0,0 +1,572 @@ +package filestore + +import ( + //"runtime/debug" + //"bytes" + "errors" + "io" + "os" + "path/filepath" + "sync" + + "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess" + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/util" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" + "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query" + dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query" +) + +var log = logging.Logger("filestore") +var Logger = log + +type VerifyWhen int + +const ( + VerifyNever VerifyWhen = iota + VerifyIfChanged + VerifyAlways +) + +type Datastore struct { + db dbwrap + verify VerifyWhen + + // updateLock should be held whenever updating the database. It + // is designed to only be held for a very short period of time and + // should not be held when doing potentially expensive operations + // such as computing a hash or any sort of I/O. + updateLock sync.Mutex + + // A snapshot of the DB the last time it was in a consistent + // state, if null than there are no outstanding adds + snapshot Snapshot + // If the snapshot was used, if not true than Release() can be + // called to help save space + snapshotUsed bool + + addLocker addLocker + + // maintenanceLock is designed to be help for a longer period + // of time. It, as it names suggests, is designed to be avoid + // race conditions during maintenance. Operations that add + // blocks are expected to already be holding the "read" lock. + // Maintaince operations will hold an exclusive lock. + //maintLock sync.RWMutex +} + +type Basic struct { + db dbread + ds *Datastore +} + +func Init(path string) error { + db, err := leveldb.OpenFile(path, &opt.Options{ + Compression: opt.NoCompression, + }) + if err != nil { + return err + } + db.Close() + return nil +} + +func New(path string, verify VerifyWhen, noCompression bool) (*Datastore, error) { + dbOpts := &opt.Options{ErrorIfMissing: true} + if noCompression { + dbOpts.Compression = opt.NoCompression + } + db, err := leveldb.OpenFile(path, dbOpts) + if err != nil { + return nil, err + } + ds := &Datastore{db: dbwrap{dbread{db}, db}, verify: verify} + ds.addLocker.ds = ds + return ds, nil +} + +func (d *Datastore) Put(key ds.Key, value interface{}) error { + dataObj, ok := value.(*DataObj) + if !ok { + return ds.ErrInvalidType + } + + if dataObj.FilePath == "" && dataObj.Size == 0 { + // special case to handle empty files + d.updateLock.Lock() + defer d.updateLock.Unlock() + return d.db.Put(HashToKey(key.String()), dataObj) + } + + // Make sure the filename is an absolute path + if !filepath.IsAbs(dataObj.FilePath) { + return errors.New("datastore put: non-absolute filename: " + dataObj.FilePath) + } + + // Make sure we can read the file as a sanity check + file, err := os.Open(dataObj.FilePath) + if err != nil { + return err + } + defer file.Close() + + // See if we have the whole file in the block + if dataObj.Offset == 0 && !dataObj.WholeFile() { + // Get the file size + info, err := file.Stat() + if err != nil { + return err + } + if dataObj.Size == uint64(info.Size()) { + dataObj.Flags |= WholeFile + } + } + + d.updateLock.Lock() + defer d.updateLock.Unlock() + + hash := HashToKey(key.String()) + have, err := d.db.Has(hash) + if err != nil { + return err + } + if !have { + // First the easy case, the hash doesn't exist yet so just + // insert the data object as is + return d.db.Put(hash, dataObj) + } + + // okay so the hash exists, see if we already have this DataObj + dbKey := NewDbKey(key.String(), dataObj.FilePath, int64(dataObj.Offset), nil) + foundKey, _, err := d.GetDirect(dbKey) + if err != nil && err != ds.ErrNotFound { + return err + } + + if err == nil { + // the DataObj already exists so just replace it + return d.db.Put(foundKey, dataObj) + } + + // the DataObj does not exist so insert it using the full key to + // avoid overwritting the existing entry + return d.db.Put(dbKey, dataObj) +} + +func (d *Datastore) Get(dsKey ds.Key) (value interface{}, err error) { + hash := HashToKey(dsKey.String()) + + // we need a consistent view of the database so take a snapshot + ss0, err := d.db.db.GetSnapshot() + if err != nil { + return nil, err + } + defer ss0.Release() + ss := dbread{ss0} + + val, err := ss.GetHash(hash.Bytes) + if err == leveldb.ErrNotFound { + return nil, ds.ErrNotFound + } else if err != nil { + return nil, err + } + data, err := GetData(d, hash, val, d.verify) + if err == nil { + return data, nil + } + + //println("GET TRYING ALTERNATIVES") + + // See if we have any other DataObj's for the same hash that are + // valid + itr := ss.GetAlternatives(hash.Bytes) + for itr.Next() { + key := itr.Key() + val, err := itr.Value() + if err != nil { + return nil, err + } + data, err = GetData(d, key, val, d.verify) + if err == nil { + // we found one + d.updateGood(hash, key, val) + return data, nil + } + if err != InvalidBlock { + return nil, err + } + } + + return nil, err +} + +func (d *Datastore) updateGood(hash *DbKey, key *DbKey, dataObj *DataObj) { + d.updateLock.Lock() + defer d.updateLock.Unlock() + bad, err := d.db.GetHash(hash.Bytes) + if err != nil { + log.Warningf("%s: updateGood: %s", key, err) + } + badKey := NewDbKey(hash.Hash, bad.FilePath, int64(bad.Offset), nil) + good, err := d.db.Get(key) + if err != nil { + log.Warningf("%s: updateGood: %s", key, err) + } + // use batching as this needs to be done in a single atomic + // operation, to avoid problems with partial failures + batch := NewBatch() + batch.Put(hash, good) + batch.Put(badKey, bad) + batch.Delete(key.Bytes) + err = d.db.Write(batch) + if err != nil { + log.Warningf("%s: updateGood: %s", key, err) + } +} + +// Get the key as a DataObj. To handle multiple DataObj per Hash a +// block can be retrieved by either by just the hash or the hash +// combined with filename and offset. +// +// In addition to the date GteDirect will return the key the block was +// found under. +func (d *Basic) GetDirect(key *DbKey) (*DbKey, *DataObj, error) { + if string(key.Bytes) != key.String() { + panic(string(key.Bytes) + " != " + key.String()) + } + val, err := d.db.Get(key) + if err != leveldb.ErrNotFound { // includes the case when err == nil + return key, val, err + } + + if key.FilePath == "" { + return nil, nil, ds.ErrNotFound + } + + hash := HashToKey(key.Hash) + return d.getIndirect(hash, key) +} + +// We have a key with filename and offset that was not found directly. +// Check to see it it was stored just using the hash. +func (d *Basic) getIndirect(hash *DbKey, key *DbKey) (*DbKey, *DataObj, error) { + val, err := d.db.GetHash(hash.Bytes) + if err == leveldb.ErrNotFound { + return nil, nil, ds.ErrNotFound + } else if err != nil { + return nil, nil, err + } + + if key.FilePath != val.FilePath || uint64(key.Offset) != val.Offset { + return nil, nil, ds.ErrNotFound + } + + return hash, val, nil +} + +func (d *Datastore) GetDirect(key *DbKey) (*DbKey, *DataObj, error) { + return d.AsBasic().GetDirect(key) +} + +type KeyVal struct { + Key *DbKey + Val *DataObj +} + +func (d *Basic) GetAll(k *DbKey) ([]KeyVal, error) { + //println("GetAll:", k.Format()) + hash := k.HashOnly() + dataObj, err := d.db.GetHash(hash.Bytes) + if err == leveldb.ErrNotFound { + return nil, ds.ErrNotFound + } else if err != nil { + return nil, err + } + //println("GetAll", k.Format(), "finding matches...") + var res []KeyVal + if haveMatch(k, dataObj) { + //println("GetAll match <0>", hash.Format(), k.Format()) + res = append(res, KeyVal{hash, dataObj}) + } + itr := d.db.GetAlternatives(hash.Bytes) + for itr.Next() { + dataObj, err = itr.Value() + if err != nil { + return nil, err + } + //println("GetAll match ???", itr.Key().Format()) + if haveMatch(k, dataObj) { + //println("GetAll match <1>", itr.Key().Format(), k.Format()) + res = append(res, KeyVal{itr.Key(), dataObj}) + } + } + return res, nil +} + +func haveMatch(k *DbKey, dataObj *DataObj) bool { + return ((k.FilePath == "" || k.FilePath == dataObj.FilePath) && + (k.Offset == -1 || uint64(k.Offset) == dataObj.Offset)) +} + +type IsPinned int + +const ( + NotPinned = 1 + MaybePinned = 2 +) + +var RequirePinCheck = errors.New("Will delete last DataObj for hash, pin check required.") + +// Delete a single DataObj +// FIXME: Needs testing! +func (d *Datastore) DelSingle(key *DbKey, isPinned IsPinned) error { + if key.FilePath != "" { + return d.DelDirect(key, isPinned) + } + d.updateLock.Lock() + defer d.updateLock.Unlock() + found, err := d.db.Has(key) + if err != nil { + return err + } else if !found { + return ds.ErrNotFound + } + + return d.doDelete(key, isPinned) +} + +// Directly delete a single DataObj based on the full key +// FIXME: Needs testing! +func (d *Datastore) DelDirect(key *DbKey, isPinned IsPinned) error { + if key.FilePath == "" && key.Offset == -1 { + panic("Cannot delete with hash only key") + return errors.New("Cannot delete with hash only key") + } + d.updateLock.Lock() + defer d.updateLock.Unlock() + found, err := d.db.Has(key) + if err != nil { + return err + } + if found { + return d.db.Delete(key.Bytes) + } + hash := NewDbKey(key.Hash, "", -1, nil) + + _, _, err = d.AsBasic().getIndirect(hash, key) + if err != nil { + return err + } + + return d.doDelete(hash, isPinned) +} + +func (d *Datastore) doDelete(hash *DbKey, isPinned IsPinned) error { + itr := d.db.GetAlternatives(hash.Bytes) + haveAlt := itr.Next() + + if isPinned == MaybePinned && !haveAlt { + return RequirePinCheck + } + + batch := NewBatch() + + batch.Delete(hash.Bytes) + if haveAlt { + val, err := itr.Value() + if err != nil { + return err + } + batch.Put(hash, val) + batch.Delete(itr.Key().Bytes) + } + return d.db.Write(batch) +} + +func (d *Datastore) Update(key *DbKey, val *DataObj) { + if key.FilePath == "" { + key = NewDbKey(key.Hash, val.FilePath, int64(val.Offset), nil) + } + d.updateLock.Lock() + defer d.updateLock.Unlock() + foundKey, _, err := d.GetDirect(key) + if err != nil { + return + } + d.db.Put(foundKey, val) +} + +var InvalidBlock = errors.New("filestore: block verification failed") +var TouchedBlock = errors.New("filestore: modtimes differ, contents may be invalid") + +// Verify as much as possible without opening the file, the result is +// a best guess. +func VerifyFast(val *DataObj) error { + // There is backing file, nothing to check + if val.HaveBlockData() { + return nil + } + + // get the file's metadata, return on error + fileInfo, err := os.Stat(val.FilePath) + if err != nil { + return err + } + + // the file has shrunk, the block invalid + if val.Offset+val.Size > uint64(fileInfo.Size()) { + return InvalidBlock + } + + // the file mtime has changes, the block is _likely_ invalid + modtime := FromTime(fileInfo.ModTime()) + if modtime != val.ModTime { + return TouchedBlock + } + + // block already marked invalid + if val.Invalid() { + return InvalidBlock + } + + // the block _seams_ ok + return nil +} + +// Get the orignal data out of the DataObj +func GetData(d *Datastore, key *DbKey, val *DataObj, verify VerifyWhen) ([]byte, error) { + if val == nil { + return nil, errors.New("Nil DataObj") + } + + // If there is no data to get from a backing file then there + // is nothing more to do so just return the block data + if val.HaveBlockData() { + return val.Data, nil + } + + invalid := val.Invalid() + + // Open the file and seek to the correct position + file, err := os.Open(val.FilePath) + if err != nil { + return nil, err + } + defer file.Close() + _, err = file.Seek(int64(val.Offset), 0) + if err != nil { + return nil, err + } + + // Reconstruct the original block, if we get an EOF + // than the file shrunk and the block is invalid + data, _, err := Reconstruct(val.Data, file, val.Size) + reconstructOk := true + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } else if err != nil { + log.Debugf("invalid block: %s: %s\n", MHash(key), err.Error()) + reconstructOk = false + invalid = true + } + + if verify == VerifyNever { + if invalid { + return nil, InvalidBlock + } else { + return data, nil + } + } + + // get the new modtime + fileInfo, err := file.Stat() + if err != nil { + return nil, err + } + modtime := FromTime(fileInfo.ModTime()) + + // Verify the block contents if required + if reconstructOk && (verify == VerifyAlways || modtime != val.ModTime) { + log.Debugf("verifying block %s\n", MHash(key)) + origKey, _ := key.Cid() + newKey, _ := origKey.Prefix().Sum(data) + invalid = !origKey.Equals(newKey) + } + + // Update the block if the metadata has changed + if invalid != val.Invalid() || modtime != val.ModTime { + log.Debugf("updating block %s\n", MHash(key)) + newVal := *val + newVal.SetInvalid(invalid) + newVal.ModTime = modtime + // ignore errors as they are nonfatal + d.Update(key, &newVal) + } + + // Finally return the result + if invalid { + log.Debugf("invalid block %s\n", MHash(key)) + return nil, InvalidBlock + } else { + return data, nil + } +} + +func (d *Datastore) Has(key ds.Key) (exists bool, err error) { + // FIXME: This is too simple + return d.db.HasHash(key.Bytes()) +} + +func (d *Datastore) Delete(key ds.Key) error { + //d.updateLock.Lock() + //defer d.updateLock.Unlock() + //return d.db.Delete(key.Bytes()) + return errors.New("Deleting filestore blocks via Delete() method is unsupported.") +} + +func (d *Datastore) Query(q query.Query) (query.Results, error) { + if (q.Prefix != "" && q.Prefix != "/") || + len(q.Filters) > 0 || + len(q.Orders) > 0 || + q.Limit > 0 || + q.Offset > 0 || + !q.KeysOnly { + // TODO this is overly simplistic, but the only caller is + // `ipfs refs local` for now, and this gets us moving. + return nil, errors.New("filestore only supports listing all keys in random order") + } + qrb := dsq.NewResultBuilder(q) + qrb.Process.Go(func(worker goprocess.Process) { + var rnge *util.Range + i := d.db.db.NewIterator(rnge, nil) + defer i.Release() + for i.Next() { + k := ds.NewKey(string(i.Key())).String() + e := dsq.Entry{Key: k} + select { + case qrb.Output <- dsq.Result{Entry: e}: // we sent it out + case <-worker.Closing(): // client told us to end early. + break + } + } + if err := i.Error(); err != nil { + select { + case qrb.Output <- dsq.Result{Error: err}: // client read our error + case <-worker.Closing(): // client told us to end. + return + } + } + }) + go qrb.Process.CloseAfterChildren() + return qrb.Results(), nil +} + +func (d *Datastore) Close() error { + return d.db.db.Close() +} + +func (d *Datastore) Batch() (ds.Batch, error) { + return ds.NewBasicBatch(d), nil +} diff --git a/filestore/dbwrap.go b/filestore/dbwrap.go new file mode 100644 index 00000000000..dbc7edb4b69 --- /dev/null +++ b/filestore/dbwrap.go @@ -0,0 +1,176 @@ +package filestore + +import ( + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/iterator" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/util" +) + +type readops interface { + Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) + Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) + NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator +} + +type dbread struct { + db readops +} + +type dbwrap struct { + dbread + db *leveldb.DB +} + +func Decode(bytes []byte) (*DataObj, error) { + val := new(DataObj) + err := val.Unmarshal(bytes) + if err != nil { + return nil, err + } + return val, nil +} + +func (w dbread) GetHash(key []byte) (*DataObj, error) { + val, err := w.db.Get(key, nil) + if err != nil { + return nil, err + } + return Decode(val) +} + +func (w dbread) Get(key *DbKey) (*DataObj, error) { + if key.FilePath == "" { + return w.GetHash(key.Bytes) + } + val, err := w.db.Get(key.Bytes, nil) + if err != nil { + return nil, err + } + dataObj, err := Decode(val) + if err != nil { + return nil, err + } + dataObj.FilePath = key.FilePath + dataObj.Offset = uint64(key.Offset) + return dataObj, err +} + +func (d dbread) GetAlternatives(key []byte) *Iterator { + start := make([]byte, 0, len(key)+1) + start = append(start, key...) + start = append(start, byte('/')) + stop := make([]byte, 0, len(key)+1) + stop = append(stop, key...) + stop = append(stop, byte('/')+1) + return &Iterator{iter: d.db.NewIterator(&util.Range{start, stop}, nil)} +} + +func (w dbread) HasHash(key []byte) (bool, error) { + return w.db.Has(key, nil) +} + +func (w dbread) Has(key *DbKey) (bool, error) { + return w.db.Has(key.Bytes, nil) +} + +func marshal(key *DbKey, val *DataObj) ([]byte, error) { + if key.FilePath != "" { + val.FilePath = "" + val.Offset = 0 + } + return val.Marshal() +} + +// Put might modify `val`, it is not safe to use the objected pointed +// by `val` after this call. +func (w dbwrap) Put(key *DbKey, val *DataObj) error { + data, err := marshal(key, val) + if err != nil { + return err + } + return w.db.Put(key.Bytes, data, nil) +} + +func (w dbwrap) Delete(key []byte) error { + return w.db.Delete(key, nil) +} + +func (w dbwrap) Write(b dbbatch) error { + return w.db.Write(b.batch, nil) +} + +type dbbatch struct { + batch *leveldb.Batch +} + +func NewBatch() dbbatch { + return dbbatch{new(leveldb.Batch)} +} + +// Put might modify `val`, it is not safe to use the objected pointed +// by `val` after this call. +func (b dbbatch) Put(key *DbKey, val *DataObj) error { + data, err := marshal(key, val) + if err != nil { + return err + } + b.batch.Put(key.Bytes, data) + return nil +} + +func (b dbbatch) Delete(key []byte) { + b.batch.Delete(key) +} + +type Iterator struct { + key *DbKey + value *DataObj + iter iterator.Iterator +} + +func (d dbread) NewIterator() *Iterator { + return &Iterator{iter: d.db.NewIterator(nil, nil)} +} + +func (itr *Iterator) Next() bool { + itr.key = nil + itr.value = nil + return itr.iter.Next() +} + +func (itr *Iterator) Key() *DbKey { + if itr.key == nil { + bytes := itr.iter.Key() + itr.key = &DbKey{ + Key: ParseDsKey(string(bytes)), + Bytes: bytes, + } + } + return itr.key +} + +func (itr *Iterator) Value() (*DataObj, error) { + if itr.value != nil { + return itr.value, nil + } + bytes := itr.iter.Value() + if bytes == nil { + return nil, nil + } + var err error + itr.value, err = Decode(bytes) + if err != nil { + return nil, err + } + key := itr.Key() + if key.FilePath != "" { + itr.value.FilePath = key.FilePath + itr.value.Offset = uint64(key.Offset) + } + return itr.value, nil +} + +func (itr *Iterator) Release() { + itr.iter.Release() +} diff --git a/filestore/examples/add-dir b/filestore/examples/add-dir new file mode 100755 index 00000000000..75f64d78a3a --- /dev/null +++ b/filestore/examples/add-dir @@ -0,0 +1,297 @@ +#!/usr/bin/python3 + +# +# This script will add or update files in a directly (recursively) +# without copying the data into the datastore. Unlike +# add-dir-simplyy it will use it's own file to keep track of what +# files are added +# +# This script will not clean out invalid entries from the filestore, +# for that you should use "filestore clean full" from time to time. +# + +import sys +import os.path +import subprocess as sp +import stat + +# +# Maximum length of command line, this may need to be lowerd on +# windows. +# + +MAX_CMD_LEN = 120 * 1024 + +def print_err(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def usage(): + print_err("Usage: ", sys.argv[0], "[--scan] DIR [CACHE]") + sys.exit(1) + +def main(): + global scan,dir,cache + # + # Parse command line arguments + # + i = 1 + + if i >= len(sys.argv): usage() + if sys.argv[i] == "--scan": + scan = True + i += 1 + else: + scan = False + + if i >= len(sys.argv): usage() + dir = sys.argv[i] + if not os.path.isabs(dir): + print_err("directory name must be absolute:", dir) + sys.exit(1) + i += 1 + + if i < len(sys.argv): + cache = sys.argv[i] + if not os.path.isabs(cache): + print_err("cache file name must be absolute:", dir) + sys.exit(1) + else: + cache = None + + # + # State variables + # + + before = [] # list of (hash mtime path) -- from data file + + file_modified = set() + hash_ok = {} + + already_have = set() + toadd = {} + + # + # Initialization + # + + if cache != None and os.path.exists(cache): + load_cache(before,hash_ok,file_modified) + os.rename(cache, cache+".old") + elif scan: + init_cache(before,hash_ok) + + # + # Open the new cache file for writing + # + + if cache == None: + f = open(os.devnull, 'w') + else: + try: + f = open(cache, 'w') + except OSError as e: + print_err("count not write to cache file: ", e) + try: + os.rename(cache+".old", cache) + except OSError: + pass + sys.exit(1) + + # + # Figure out what files don't need to be readded and write them + # out to the cache. + # + + for hash,mtime,path in before: + if hash_ok.get(hash, True) == False or path in file_modified: + # if the file still exists it will be picked up in the + # directory scan so no need to do anything special + pass + else: + already_have.add(path) + print(hash,mtime,path, file=f) + + # To cut back on memory usage + del before + del file_modified + del hash_ok + + # + # Figure out what files need to be re-added + # + + print("checking for files to add...") + for root, dirs, files in os.walk(dir): + for file in files: + try: + path = os.path.join(root,file) + if path not in already_have: + if not os.access(path, os.R_OK): + print_err("SKIPPING", path, ":", "R_OK access check failed") + continue + finf = os.stat(path, follow_symlinks=False) + if not stat.S_ISREG(finf.st_mode): + continue + mtime = "%.3f" % finf.st_mtime + #print("will add", path) + toadd[path] = mtime + except OSError as e: + print_err("SKIPPING", path, ":", e) + + # + # Finally, do the add. Write results to the cache file as they are + # added. + # + + print("adding", len(toadd), "files...") + + errors = False + + class FilestoreAdd(Xargs): + def __init__(self, args): + Xargs.__init__(self, ['ipfs', 'filestore', 'add'], args) + def process_ended(self, returncode): + print("added", self.args_used, "files, ", len(self.args), "more to go.") + + for line in FilestoreAdd(list(toadd.keys())): + try: + _, hash, path = line.rstrip('\n').split(None, 2) + mtime = toadd[path] + del toadd[path] + print(hash,mtime,path, file=f) + except Exception as e: + errors = True + print_err("WARNING: problem when adding: ", path, ":", e) + # don't abort, non-fatal error + + if len(toadd) != 0: + errors = True + i = 0 + limit = 10 + for path in toadd.keys(): + print_err("WARNING:", path, "not added.") + i += 1 + if i == limit: break + if i == limit: + print_err("WARNING:", len(toadd)-limit, "additional paths(s) not added.") + + + # + # Cleanup + # + + f.close() + + if errors: + sys.exit(1) + +def load_cache(before, hash_ok, file_modified): + # + # Read in cache (if it exists) and determine any files that have modified + # + print("checking for modified files...") + try: + f = open(cache) + except OSError as e: + print_err("count not open cache file: ", e) + sys.exit(1) + for line in f: + hash,mtime,path = line.rstrip('\n').split(' ', 2) + try: + new_mtime = "%.3f" % os.path.getmtime(path) + except OSError as e: + print_err("skipping:", path, ":", e.strerror) + continue + before.append((hash,mtime,path),) + if mtime != new_mtime: + print("file modified:", path) + file_modified.add(path) + hash_ok[hash] = None + del f + + # + # Determine any hashes that have become invalid. All files with + # that hash will then be readded in an attempt to fix it. + # + print("checking for invalid hashes...") + for line in Xargs(['ipfs', 'filestore', 'verify', '-v2', '-l3', '--porcelain'], list(hash_ok.keys())): + line = line.rstrip('\n') + _, status, hash, path = line.split('\t') + hash_ok[hash] = status == "ok" or status == "appended" or status == "found" + if not hash_ok[hash]: + print("hash not ok:", status,hash,path) + + for hash,val in hash_ok.items(): + if val == None: + print_err("WARNING: hash status unknown: ", hash) + +def init_cache(before, hash_ok): + # + # Use what is in the filestore already to initialize the cache file + # + print("scanning filestore for files already added...") + for line in Xargs(['ipfs', 'filestore', 'verify', '-v4', '-l3', '--porcelain'], [os.path.join(dir,'')]): + line = line.rstrip('\n') + what, status, hash, path = line.split('\t') + if what == "root" and status == "ok": + try: + mtime = "%.3f" % os.path.getmtime(path) + except OSError as e: + print_err("skipping:", path, ":", e.strerror) + continue + hash_ok[hash] = True + before.append((hash,mtime,path),) + +class Xargs: + def __init__(self, cmd, args): + self.cmd = cmd + self.args = args + self.pipe = None + self.args_used = -1 + + def __iter__(self): + return self + + def __next__(self): + if self.pipe == None: + self.launch() + if self.pipe == None: + raise StopIteration() + line = self.pipe.stdout.readline() + if line == '': + self.close() + return self.__next__() + return line + + def launch(self): + if len(self.args) == 0: + return + cmd_len = len(' '.join(self.cmd)) + 1 + i = 0 + while i < len(self.args): + cmd_len += len(self.args[i]) + 1 + if cmd_len > MAX_CMD_LEN: break + i += 1 + cmd = self.cmd + self.args[0:i] + self.args_used = i + self.args = self.args[i:] + self.pipe = sp.Popen(cmd, stdout=sp.PIPE, bufsize=-1, universal_newlines=True) + + def close(self): + pipe = self.pipe + pipe.stdout.close() + pipe.wait() + + self.process_ended(pipe.returncode) + + if pipe.returncode < 0: + raise sp.CalledProcessError(returncode=pipe.returncode, cmd=pipe.args) + + self.pipe = None + + def process_ended(self, returncode): + pass + +if __name__ == "__main__": + main() + diff --git a/filestore/examples/add-dir-simple.sh b/filestore/examples/add-dir-simple.sh new file mode 100755 index 00000000000..cfae49e4c6a --- /dev/null +++ b/filestore/examples/add-dir-simple.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +# +# This script will add or update files in a directly (recursively) +# without copying the data into the datastore. When run the first +# time it will add all the files. When run the again it will readd +# any modified or new files. Invalid blocks due to changed or removed +# files will be cleaned out. +# +# NOTE: Zero length files will always be readded. +# + +# Exit on any error +set -e + +LC_ALL=C + +if [ "$#" -ne 1 ]; then + echo "usage: $0 DIR" + exit 1 +fi + +DIR="$1" + +# +# Creating a tmp directory to store our scratch files +# +# Comment the trap to keep the directory around for debugging +# +WKDIR="`mktemp -d -t filestore.XXXXXX`" +#echo $WKDIR +trap "rm -r '$WKDIR'" EXIT + +cd "$WKDIR" + +# +# A version of xargs that will do nothing if there is no output. The +# "_r" comes from the non-posix "-r" option from GNU xargs. +# +xargs_r () { + TMP="`mktemp`" + cat > "$TMP" + if [ -s "$TMP" ] + then + cat "$TMP" | xargs "$@" + fi + rm "$TMP" +} + +# +# This function will run "filestore verify" but only on the files +# under "$DIR". +# +verify() { + ipfs filestore verify --porcelain "$@" "$DIR"/ +} + +# +# First figure out what we already have in the filestore +# +verify --level=2 > verify.res 2> verify.err + +# Get a list of files that need to be updated +cat verify.res | awk -F'\t' '$2 != "ok" {print $4}' | sort -u > verify.notok + +# Get a list of all files in the filestore +cat verify.res | cut -f4 | sort -u > prev-files + +# +# Now figure out what we have in the filesystem +# +find "$DIR" -type f | sort -u > cur-files + +# Get a list of changed files +comm -12 verify.notok cur-files > changed-files + +# Get a list of new files to add +comm -13 prev-files cur-files > new-files + +# +# Readd any changed or new files +# +cat changed-files new-files | xargs_r -d '\n' ipfs filestore add + +# +# Manually clean the filestore. Done manually so we only clean he +# files under $DIR +# +# Step 1: remove bad blocks +verify -v6 \ + | tee verify2.res \ + | awk '$2 == "changed" || $2 == "no-file" {print $3}' \ + | xargs_r ipfs filestore rm --direct --force + +# Step 2: remove incomplete files, the "-l0" is important as it tells +# us not to try and verify individual blocks just list root nodes +# that are now incomplete. +verify -v2 -l0 \ + | tee verify3.res \ + | awk '$2 == "incomplete" {print $3}' \ + | xargs_r ipfs filestore rm --direct + diff --git a/filestore/key.go b/filestore/key.go new file mode 100644 index 00000000000..1e493f39340 --- /dev/null +++ b/filestore/key.go @@ -0,0 +1,187 @@ +package filestore + +import ( + "bytes" + "fmt" + "strconv" + "strings" + + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + base32 "gx/ipfs/Qmb1DA2A9LS2wR4FFweB4uEDomFsdmnw1VLawLE1yQzudj/base32" +) + +type Key struct { + Hash string + FilePath string // empty string if not given + Offset int64 // -1 if not given +} + +func ParseKey(str string) (*DbKey, error) { + idx := strings.Index(str, "/") + var key *DbKey + if idx == -1 { + idx = len(str) + } + if idx != 0 { // we have a Hash + mhash := str[:idx] + c, err := cid.Decode(mhash) + if err != nil { + return nil, err + } + key = CidToKey(c) + } else { + key = &DbKey{} + } + if idx == len(str) { // we just have a hash + return key, nil + } + str = str[idx+1:] + parseRest(&key.Key, str) + key.Bytes = key.Key.Bytes() + return key, nil +} + +func ParseDsKey(str string) Key { + idx := strings.Index(str[1:], "/") + 1 + if idx == 0 { + return Key{str, "", -1} + } + key := Key{Hash: str[:idx]} + str = str[idx+1:] + parseRest(&key, str) + return key +} + +func parseRest(key *Key, str string) { + filename := strings.Trim(str, "0123456789") + if len(filename) <= 2 || filename[len(filename)-2:] != "//" || len(str) == len(filename) { + key.FilePath = filename + key.Offset = -1 + return + } + offsetStr := str[len(filename):] + key.FilePath = filename[:len(filename)-2] + key.Offset, _ = strconv.ParseInt(offsetStr, 10, 64) +} + +func (k Key) String() string { + str := k.Hash + if k.FilePath == "" { + return str + } + str += "/" + str += k.FilePath + if k.Offset == -1 { + return str + } + str += "//" + str += strconv.FormatInt(k.Offset, 10) + return str +} + +func (k Key) Bytes() []byte { + if k.FilePath == "" { + return []byte(k.Hash) + } + buf := bytes.NewBuffer(nil) + if k.Offset == -1 { + fmt.Fprintf(buf, "%s/%s", k.Hash, k.FilePath) + } else { + fmt.Fprintf(buf, "%s/%s//%d", k.Hash, k.FilePath, k.Offset) + } + return buf.Bytes() +} + +func (k Key) Cid() (*cid.Cid, error) { + binary, err := base32.RawStdEncoding.DecodeString(k.Hash[1:]) + if err != nil { + return nil, err + } + return cid.Cast(binary) +} + +type DbKey struct { + Key + Bytes []byte + cid *cid.Cid +} + +func ParseDbKey(key string) *DbKey { + return &DbKey{ + Key: ParseDsKey(key), + Bytes: []byte(key), + } +} + +func NewDbKey(hash string, filePath string, offset int64, cid *cid.Cid) *DbKey { + key := &DbKey{Key: Key{hash, filePath, offset}, cid: cid} + key.Bytes = key.Key.Bytes() + return key +} + +func KeyToKey(key Key) *DbKey { + return &DbKey{key, key.Bytes(), nil} +} + +func HashToKey(hash string) *DbKey { + return NewDbKey(hash, "", -1, nil) +} + +func CidToKey(c *cid.Cid) *DbKey { + return NewDbKey(dshelp.CidToDsKey(c).String(), "", -1, c) +} + +func (k *DbKey) HashOnly() *DbKey { + if k.cid != nil { + return CidToKey(k.cid) + } else { + return HashToKey(k.Hash) + } +} + +func (k *DbKey) Cid() (*cid.Cid, error) { + if k.cid == nil { + var err error + k.cid, err = k.Key.Cid() + if err != nil { + return nil, err + } + } + return k.cid, nil +} + +type havecid interface { + Cid() (*cid.Cid, error) +} + +func MHash(k havecid) string { + key, err := k.Cid() + if err != nil { + return "??????????????????????????????????????????????" + } + return key.String() +} + +func (k Key) Format() string { + if k.FilePath == "" { + return MHash(k) + } + return Key{MHash(k), k.FilePath, k.Offset}.String() +} + +func (k *DbKey) Format() string { + mhash := "" + if k.Hash != "" { + mhash = MHash(k) + } + if k.FilePath == "" { + return mhash + } + return Key{mhash, k.FilePath, k.Offset}.String() +} + +func (k *DbKey) MakeFull(dataObj *DataObj) *DbKey { + newKey := Key{k.Hash, dataObj.FilePath, int64(dataObj.Offset)} + return &DbKey{newKey, newKey.Bytes(), k.cid} +} diff --git a/filestore/key_test.go b/filestore/key_test.go new file mode 100644 index 00000000000..c76ae721ee2 --- /dev/null +++ b/filestore/key_test.go @@ -0,0 +1,48 @@ +package filestore + +import ( + "testing" +) + +func testParse(t *testing.T, str string, expect Key) { + res,err := ParseKey(str) + if err != nil { + t.Errorf("%s", err) + } + if res.Key != expect { + t.Errorf("parse failed on: %s: %#v != %#v", str, expect, res.Key) + } + if str != res.Format() { + t.Errorf("Format() format failed %s != %s", str, res.Format()) + } +} + +func testDsParse(t *testing.T, str string, expect Key) { + res := ParseDsKey(str) + if res != expect { + t.Errorf("parse failed on: %s", str) + } + if str != res.String() { + t.Errorf("String() format failed %s != %s", str, res.String()) + } + if str != string(res.Bytes()) { + t.Errorf("Bytes() format failed %s != %s", str, res.String()) + } +} + +func TestKey(t *testing.T) { + qmHash := "/CIQPJLLZXHBPDKSP325GP7BLB6J3WNGKMDZJWZRGANTAN22QKXDNY6Y" + zdHash := "/AFZBEIGD4KVH2JPQABBLQGN44DZVK5F3WWBEFEUDWFZ2ANB3PLOXSHWTDY" + testParse(t, "QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp", Key{qmHash, "", -1}) + testParse(t, "zdvgqEbdrK4PzARFB7twNKangqFF3mgWeuJJAtMUwdDwFq7Pj", Key{zdHash, "", -1}) + testParse(t, "QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp/dir/file", Key{qmHash, "dir/file", -1}) + testParse(t, "QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp//dir/file", Key{qmHash, "/dir/file", -1}) + testParse(t, "QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp//dir/file//23", Key{qmHash, "/dir/file", 23}) + testParse(t, "//just/a/file", Key{"", "/just/a/file", -1}) + testParse(t, "/just/a/file", Key{"", "just/a/file", -1}) + + testDsParse(t, "/ED65SD", Key{"/ED65SD", "", -1}) + testDsParse(t, "/ED65SD//some/file", Key{"/ED65SD", "/some/file", -1}) + testDsParse(t, "/ED65SD//some/file//34", Key{"/ED65SD", "/some/file", 34}) + testDsParse(t, "/ED65SD/c:/some/file//34", Key{"/ED65SD", "c:/some/file", 34}) +} diff --git a/filestore/pb/Makefile b/filestore/pb/Makefile new file mode 100644 index 00000000000..4b6a1d37569 --- /dev/null +++ b/filestore/pb/Makefile @@ -0,0 +1,10 @@ +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) + +all: $(GO) + +%.pb.go: %.proto + protoc --gofast_out=. $< + +clean: + rm *.pb.go diff --git a/filestore/pb/dataobj.pb.go b/filestore/pb/dataobj.pb.go new file mode 100644 index 00000000000..fe7f685473a --- /dev/null +++ b/filestore/pb/dataobj.pb.go @@ -0,0 +1,499 @@ +// Code generated by protoc-gen-gogo. +// source: dataobj.proto +// DO NOT EDIT! + +/* + Package datastore_pb is a generated protocol buffer package. + + It is generated from these files: + dataobj.proto + + It has these top-level messages: + DataObj +*/ +package datastore_pb + +import proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" +import fmt "fmt" +import math "math" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type DataObj struct { + FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"` + Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"` + Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"` + Data []byte `protobuf:"bytes,4,opt,name=Data" json:"Data,omitempty"` + Flags *uint64 `protobuf:"varint,8,opt,name=Flags" json:"Flags,omitempty"` + Modtime *float64 `protobuf:"fixed64,9,opt,name=Modtime" json:"Modtime,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *DataObj) Reset() { *m = DataObj{} } +func (m *DataObj) String() string { return proto.CompactTextString(m) } +func (*DataObj) ProtoMessage() {} + +func (m *DataObj) GetFilePath() string { + if m != nil && m.FilePath != nil { + return *m.FilePath + } + return "" +} + +func (m *DataObj) GetOffset() uint64 { + if m != nil && m.Offset != nil { + return *m.Offset + } + return 0 +} + +func (m *DataObj) GetSize_() uint64 { + if m != nil && m.Size_ != nil { + return *m.Size_ + } + return 0 +} + +func (m *DataObj) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *DataObj) GetFlags() uint64 { + if m != nil && m.Flags != nil { + return *m.Flags + } + return 0 +} + +func (m *DataObj) GetModtime() float64 { + if m != nil && m.Modtime != nil { + return *m.Modtime + } + return 0 +} + +func init() { + proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj") +} +func (m *DataObj) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *DataObj) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.FilePath != nil { + data[i] = 0xa + i++ + i = encodeVarintDataobj(data, i, uint64(len(*m.FilePath))) + i += copy(data[i:], *m.FilePath) + } + if m.Offset != nil { + data[i] = 0x10 + i++ + i = encodeVarintDataobj(data, i, uint64(*m.Offset)) + } + if m.Size_ != nil { + data[i] = 0x18 + i++ + i = encodeVarintDataobj(data, i, uint64(*m.Size_)) + } + if m.Data != nil { + data[i] = 0x22 + i++ + i = encodeVarintDataobj(data, i, uint64(len(m.Data))) + i += copy(data[i:], m.Data) + } + if m.Flags != nil { + data[i] = 0x40 + i++ + i = encodeVarintDataobj(data, i, uint64(*m.Flags)) + } + if m.Modtime != nil { + data[i] = 0x49 + i++ + i = encodeFixed64Dataobj(data, i, uint64(math.Float64bits(*m.Modtime))) + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeFixed64Dataobj(data []byte, offset int, v uint64) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + data[offset+4] = uint8(v >> 32) + data[offset+5] = uint8(v >> 40) + data[offset+6] = uint8(v >> 48) + data[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Dataobj(data []byte, offset int, v uint32) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintDataobj(data []byte, offset int, v uint64) int { + for v >= 1<<7 { + data[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + data[offset] = uint8(v) + return offset + 1 +} +func (m *DataObj) Size() (n int) { + var l int + _ = l + if m.FilePath != nil { + l = len(*m.FilePath) + n += 1 + l + sovDataobj(uint64(l)) + } + if m.Offset != nil { + n += 1 + sovDataobj(uint64(*m.Offset)) + } + if m.Size_ != nil { + n += 1 + sovDataobj(uint64(*m.Size_)) + } + if m.Data != nil { + l = len(m.Data) + n += 1 + l + sovDataobj(uint64(l)) + } + if m.Flags != nil { + n += 1 + sovDataobj(uint64(*m.Flags)) + } + if m.Modtime != nil { + n += 9 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovDataobj(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozDataobj(x uint64) (n int) { + return sovDataobj(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *DataObj) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DataObj: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DataObj: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FilePath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDataobj + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(data[iNdEx:postIndex]) + m.FilePath = &s + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Offset = &v + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Size_ = &v + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthDataobj + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append([]byte{}, data[iNdEx:postIndex]...) + iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Flags", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDataobj + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Flags = &v + case 9: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Modtime", wireType) + } + var v uint64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + iNdEx += 8 + v = uint64(data[iNdEx-8]) + v |= uint64(data[iNdEx-7]) << 8 + v |= uint64(data[iNdEx-6]) << 16 + v |= uint64(data[iNdEx-5]) << 24 + v |= uint64(data[iNdEx-4]) << 32 + v |= uint64(data[iNdEx-3]) << 40 + v |= uint64(data[iNdEx-2]) << 48 + v |= uint64(data[iNdEx-1]) << 56 + v2 := float64(math.Float64frombits(v)) + m.Modtime = &v2 + default: + iNdEx = preIndex + skippy, err := skipDataobj(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDataobj + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipDataobj(data []byte) (n int, err error) { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if data[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthDataobj + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowDataobj + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipDataobj(data[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthDataobj = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowDataobj = fmt.Errorf("proto: integer overflow") +) diff --git a/filestore/pb/dataobj.proto b/filestore/pb/dataobj.proto new file mode 100644 index 00000000000..f2d9e1033de --- /dev/null +++ b/filestore/pb/dataobj.proto @@ -0,0 +1,14 @@ +package datastore.pb; + +message DataObj { + optional string FilePath = 1; + optional uint64 Offset = 2; + optional uint64 Size = 3; + optional bytes Data = 4; + + // fields 5 - 7 where used in the dev. version, best not to resuse them + + optional uint64 Flags = 8; + optional double Modtime = 9; +} + diff --git a/filestore/reconstruct.go b/filestore/reconstruct.go new file mode 100644 index 00000000000..79584f1440e --- /dev/null +++ b/filestore/reconstruct.go @@ -0,0 +1,362 @@ +package filestore + +import ( + //"bytes" + //"encoding/hex" + "errors" + "fmt" + "io" + + dag_pb "github.com/ipfs/go-ipfs/merkledag/pb" + fs_pb "github.com/ipfs/go-ipfs/unixfs/pb" + proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" +) + +type UnixFSInfo struct { + Type fs_pb.Data_DataType + Data []byte + FileSize uint64 +} + +const useFastReconstruct = true + +func Reconstruct(data []byte, in io.Reader, blockDataSize uint64) ([]byte, *UnixFSInfo, error) { + // if blockDataSize == 0 { + // res1, fsinfo1, err1 := reconstruct(data, nil) + // if err1 != nil { + // return res1, fsinfo1, err1 + // } + // _ = fsinfo1 + // res2, fsinfo2, err2 := reconstructDirect(data, nil, 0) + // _ = fsinfo2 + // if err2 != nil { + // panic(err2) + // } + // if !bytes.Equal(res1, res2) { + // println("res1") + // print(hex.Dump(res1)) + // println("res2") + // print(hex.Dump(res2)) + // panic("Result not equal!") + // } + // return res2, fsinfo2, err2 + // } + if data == nil { // we have a raw node + blockData, err := readFromFile(in, blockDataSize) + if err != nil { + println(err) + } + return blockData, nil, err + } else if useFastReconstruct { + return reconstructDirect(data, in, blockDataSize) + } else { + blockData, err := readFromFile(in, blockDataSize) + if err != nil { + return nil, nil, err + } + return reconstruct(data, blockData) + } +} + +func readFromFile(in io.Reader, blockDataSize uint64) ([]byte, error) { + var blockData []byte + if blockDataSize > 0 { + blockData = make([]byte, blockDataSize) + _, err := io.ReadFull(in, blockData) + if err != nil { + return nil, err + } + } + return blockData, nil +} + +func reconstruct(data []byte, blockData []byte) ([]byte, *UnixFSInfo, error) { + // Decode data to merkledag protobuffer + var pbn dag_pb.PBNode + err := pbn.Unmarshal(data) + if err != nil { + panic(err) + } + + // Decode node's data to unixfs protobuffer + fs_pbn := new(fs_pb.Data) + err = proto.Unmarshal(pbn.Data, fs_pbn) + if err != nil { + panic(err) + } + + // gather some data about the unixfs object + fsinfo := &UnixFSInfo{Type: *fs_pbn.Type, Data: fs_pbn.Data} + if fs_pbn.Filesize != nil { + fsinfo.FileSize = *fs_pbn.Filesize + } + + // if we won't be replasing anything no need to reencode, just + // return the original data + if fs_pbn.Data == nil && blockData == nil { + return data, fsinfo, nil + } + + fs_pbn.Data = blockData + + // Reencode unixfs protobuffer + pbn.Data, err = proto.Marshal(fs_pbn) + if err != nil { + panic(err) + } + + // Reencode merkledag protobuffer + encoded, err := pbn.Marshal() + if err != nil { + return nil, fsinfo, err + } + return encoded, fsinfo, nil +} + +type header struct { + id int32 + // An "id" of 0 indicates a message we don't care about the + // value. As we don't care about the value multiple + // fields may be concatenated into one. + wire int32 + // "wire" is the Protocol Buffer wire format + val uint64 + // The exact meaning of "val" depends on the wire format: + // if a varint (wire format 0) then val is the value of the + // variable int; if length-delimited (wire format 2) + // then val is the payload size; otherwise, val is unused. +} + +type field struct { + header + offset int + // "offset" is the offset from the start of the buffer that + // contains the protocol key-value pair corresponding to the + // field, the end of the field is the same as the offset of + // the next field. An dummy field is added at the end that + // contains the final offset (i.e. the length of the buffer) + // to avoid special cases. +} + +type fields struct { + byts []byte + flds []field +} + +func (f fields) data(i int) []byte { + return f.byts[f.flds[i].offset:f.flds[i+1].offset] +} + +func (f fields) size(i int) int { + return f.flds[i+1].offset - f.flds[i].offset +} + +func (f fields) field(i int) field { + return f.flds[i] +} + +func (f fields) fields() []field { + return f.flds[0 : len(f.flds)-1] +} + +// only valid for the length-delimited (2) wire format +func (f fields) payload(i int) []byte { + return f.byts[f.flds[i+1].offset-int(f.flds[i].val) : f.flds[i+1].offset] +} + +const ( + unixfsTypeField = 1 + unixfsDataField = 2 + unixfsFilesizeField = 3 +) + +// An implementation of reconstruct that avoids expensive +// intermertaint data structures and unnecessary copying of data by +// reading the protocol buffer messages directly. +func reconstructDirect(data []byte, blockData io.Reader, blockDataSize uint64) ([]byte, *UnixFSInfo, error) { + dag, err := decodePB(data, func(typ int32) bool { + return typ == 1 + }) + var fs fields + if err != nil { + return nil, nil, err + } + dagSz := 0 + for i, fld := range dag.fields() { + if fld.id == 1 { + fs, err = decodePB(dag.payload(i), func(typ int32) bool { + return typ == unixfsTypeField || typ == unixfsDataField || typ == unixfsFilesizeField + }) + if err != nil { + return nil, nil, err + } + } else { + dagSz += dag.size(i) + } + } + + fsinfo := new(UnixFSInfo) + if len(fs.fields()) == 0 { + return nil, nil, errors.New("no UnixFS data") + } + if fs.field(0).id != unixfsTypeField { + return nil, nil, errors.New("unexpected field order") + } else { + fsinfo.Type = fs_pb.Data_DataType(fs.field(0).val) + } + fsSz := 0 + for i, fld := range fs.fields() { + if fld.id == unixfsDataField { + if i != 1 { + return nil, nil, errors.New("unexpected field order") + } + continue + } + if fld.id == unixfsFilesizeField { + fsinfo.FileSize = fld.val + } + fsSz += fs.size(i) + } + if len(fs.fields()) >= 2 && fs.field(1).id == unixfsDataField { + fsinfo.Data = fs.payload(1) + } else if blockDataSize == 0 { + // if we won't be replasing anything no need to + // reencode, just return the original data + return data, fsinfo, nil + } + if blockDataSize > 0 { + fsSz += 1 /* header */ + sizeVarint(blockDataSize) + int(blockDataSize) + } + dagSz += 1 /* header */ + sizeVarint(uint64(fsSz)) + fsSz + + // now reencode + + out := make([]byte, 0, dagSz) + + for i, fld := range dag.fields() { + if fld.id == 1 { + out = append(out, dag.data(i)[0]) + out = append(out, proto.EncodeVarint(uint64(fsSz))...) + out, err = reconstructUnixfs(out, fs, blockData, blockDataSize) + if err != nil { + return nil, fsinfo, err + } + } else { + out = append(out, dag.data(i)...) + } + } + + if dagSz != len(out) { + return nil, nil, fmt.Errorf("verification Failed: computed-size(%d) != actual-size(%d)", dagSz, len(out)) + } + return out, fsinfo, nil +} + +func reconstructUnixfs(out []byte, fs fields, blockData io.Reader, blockDataSize uint64) ([]byte, error) { + // copy first field + out = append(out, fs.data(0)...) + + // insert Data field + if blockDataSize > 0 { + out = append(out, byte((unixfsDataField<<3)|2)) + out = append(out, proto.EncodeVarint(blockDataSize)...) + + origLen := len(out) + out = out[:origLen+int(blockDataSize)] + _, err := io.ReadFull(blockData, out[origLen:]) + if err != nil { + return out, err + } + } + + // copy rest of protocol buffer + sz := len(fs.fields()) + for i := 1; i < sz; i += 1 { + if fs.field(i).id == unixfsDataField { + continue + } + out = append(out, fs.data(i)...) + } + + return out, nil +} + +func decodePB(data []byte, keep func(int32) bool) (fields, error) { + res := make([]field, 0, 6) + offset := 0 + for offset < len(data) { + hdr, newOffset, err := getField(data, offset) + if err != nil { + return fields{}, err + } + if !keep(hdr.id) { + if len(res) > 1 && res[len(res)-1].id == 0 { + // nothing to do + // field will get merged into previous field + } else { + // set the header id to 0 to indicate + // we don't care about the value + res = append(res, field{offset: offset}) + } + } else { + res = append(res, field{hdr, offset}) + } + offset = newOffset + } + if offset != len(data) { + return fields{}, fmt.Errorf("protocol buffer sanity check failed") + } + // insert dummy field with the final offset + res = append(res, field{offset: offset}) + return fields{data, res}, nil +} + +func getField(data []byte, offset0 int) (hdr header, offset int, err error) { + offset = offset0 + hdrVal, varintSz := proto.DecodeVarint(data[offset:]) + if varintSz == 0 { + err = io.ErrUnexpectedEOF + return + } + offset += varintSz + hdr.id = int32(hdrVal) >> 3 + hdr.wire = int32(hdrVal) & 0x07 + switch hdr.wire { + case 0: // Variant + hdr.val, varintSz = proto.DecodeVarint(data[offset:]) + if varintSz == 0 { + err = io.ErrUnexpectedEOF + return + } + offset += varintSz + case 1: // 64 bit + offset += 8 + case 2: // Length-delimited + hdr.val, varintSz = proto.DecodeVarint(data[offset:]) + if varintSz == 0 { + err = io.ErrUnexpectedEOF + return + } + offset += varintSz + int(hdr.val) + case 5: // 32 bit + offset += 4 + default: + err = errors.New("unhandled wire type") + return + } + return +} + +// Note: this is copy and pasted from proto/encode.go, newer versions +// have this function exported. Once upgraded the exported function +// should be used instead. +func sizeVarint(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} diff --git a/filestore/snapshot.go b/filestore/snapshot.go new file mode 100644 index 00000000000..2dfdf05a8d0 --- /dev/null +++ b/filestore/snapshot.go @@ -0,0 +1,86 @@ +package filestore + +import ( + "sync" + + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb" +) + +type Snapshot struct { + *Basic +} + +func (ss Snapshot) Defined() bool { return ss.Basic != nil } + +func (b *Basic) Verify() VerifyWhen { return b.ds.verify } + +func (d *Basic) DB() dbread { return d.db } + +func (d *Datastore) DB() dbwrap { return d.db } + +func (d *Datastore) AsBasic() *Basic { return &Basic{d.db.dbread, d} } + +func (d *Basic) AsFull() *Datastore { return d.ds } + +func (d *Datastore) GetSnapshot() (Snapshot, error) { + if d.snapshot.Defined() { + d.snapshotUsed = true + return d.snapshot, nil + } + ss, err := d.db.db.GetSnapshot() + if err != nil { + return Snapshot{}, err + } + return Snapshot{&Basic{dbread{ss}, d}}, nil +} + +func (d *Datastore) releaseSnapshot() { + if !d.snapshot.Defined() { + return + } + if !d.snapshotUsed { + d.snapshot.db.db.(*leveldb.Snapshot).Release() + } + d.snapshot = Snapshot{} +} + +func NoOpLocker() sync.Locker { + return noopLocker{} +} + +type noopLocker struct{} + +func (l noopLocker) Lock() {} + +func (l noopLocker) Unlock() {} + +type addLocker struct { + adders int + lock sync.Mutex + ds *Datastore +} + +func (l *addLocker) Lock() { + l.lock.Lock() + defer l.lock.Unlock() + if l.adders == 0 { + l.ds.releaseSnapshot() + l.ds.snapshot, _ = l.ds.GetSnapshot() + } + l.adders += 1 + log.Debugf("acquired add-lock refcnt now %d\n", l.adders) +} + +func (l *addLocker) Unlock() { + l.lock.Lock() + defer l.lock.Unlock() + l.adders -= 1 + if l.adders == 0 { + l.ds.releaseSnapshot() + } + log.Debugf("released add-lock refcnt now %d\n", l.adders) +} + +func (d *Datastore) AddLocker() sync.Locker { + return &d.addLocker +} diff --git a/filestore/support/blockstore.go b/filestore/support/blockstore.go new file mode 100644 index 00000000000..a89fd4c2a97 --- /dev/null +++ b/filestore/support/blockstore.go @@ -0,0 +1,145 @@ +package filestore_support + +import ( + "fmt" + b "github.com/ipfs/go-ipfs/blocks" + BS "github.com/ipfs/go-ipfs/blocks/blockstore" + . "github.com/ipfs/go-ipfs/filestore" + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + pi "github.com/ipfs/go-ipfs/thirdparty/posinfo" + fs_pb "github.com/ipfs/go-ipfs/unixfs/pb" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" + + "gx/ipfs/QmRpAnJ1Mvd2wCtwoFevW8pbLTivUqmFxynptG6uvp1jzC/safepath" +) + +type blockstore struct { + BS.GCBlockstore + filestore *Datastore +} + +func NewBlockstore(b BS.GCBlockstore, fs *Datastore) BS.GCBlockstore { + return &blockstore{b, fs} +} + +func (bs *blockstore) Put(block b.Block) error { + k := dshelp.CidToDsKey(block.Cid()) + + data, err := bs.prepareBlock(k, block) + if err != nil { + return err + } else if data == nil { + return bs.GCBlockstore.Put(block) + } + return bs.filestore.Put(k, data) +} + +func (bs *blockstore) PutMany(blocks []b.Block) error { + var nonFilestore []b.Block + + t, err := bs.filestore.Batch() + if err != nil { + return err + } + + for _, b := range blocks { + k := dshelp.CidToDsKey(b.Cid()) + data, err := bs.prepareBlock(k, b) + if err != nil { + return err + } else if data == nil { + nonFilestore = append(nonFilestore, b) + continue + } + + err = t.Put(k, data) + if err != nil { + return err + } + } + + err = t.Commit() + if err != nil { + return err + } + + if len(nonFilestore) > 0 { + err := bs.GCBlockstore.PutMany(nonFilestore) + if err != nil { + return err + } + return nil + } else { + return nil + } +} + +func (bs *blockstore) prepareBlock(k ds.Key, block b.Block) (*DataObj, error) { + altData, fsInfo, err := decode(block) + if err != nil { + return nil, err + } + + var fileSize uint64 + if fsInfo == nil { + fileSize = uint64(len(block.RawData())) + } else { + fileSize = fsInfo.FileSize + } + + if fsInfo != nil && fsInfo.Type != fs_pb.Data_Raw && fsInfo.Type != fs_pb.Data_File { + // If the node does not contain file data store using + // the normal datastore and not the filestore. + return nil, nil + } else if fileSize == 0 { + // Special case for empty files as the block doesn't + // have any file information associated with it + return &DataObj{ + FilePath: "", + Offset: 0, + Size: 0, + ModTime: 0, + Flags: Internal | WholeFile, + Data: block.RawData(), + }, nil + } else { + fsn, ok := block.(*pi.FilestoreNode) + if !ok { + return nil, fmt.Errorf("%s: no file information for block", block.Cid()) + } + posInfo := fsn.PosInfo + if posInfo.Stat == nil { + return nil, fmt.Errorf("%s: %s: no stat information for file", block.Cid(), posInfo.FullPath) + } + d := &DataObj{ + FilePath: safepath.Clean(posInfo.FullPath), + Offset: posInfo.Offset, + Size: uint64(fileSize), + ModTime: FromTime(posInfo.Stat.ModTime()), + } + if fsInfo == nil { + d.Flags |= NoBlockData + d.Data = nil + } else if len(fsInfo.Data) == 0 { + d.Flags |= Internal + d.Data = block.RawData() + } else { + d.Flags |= NoBlockData + d.Data = altData + } + return d, nil + } + +} + +func decode(block b.Block) ([]byte, *UnixFSInfo, error) { + switch block.Cid().Type() { + case cid.Protobuf: + return Reconstruct(block.RawData(), nil, 0) + case cid.Raw: + return nil, nil, nil + default: + return nil, nil, fmt.Errorf("unsupported block type") + } +} diff --git a/filestore/support/dagservice.go b/filestore/support/dagservice.go new file mode 100644 index 00000000000..903c87b77af --- /dev/null +++ b/filestore/support/dagservice.go @@ -0,0 +1,51 @@ +package filestore_support + +import ( + "context" + + . "github.com/ipfs/go-ipfs/filestore" + + dag "github.com/ipfs/go-ipfs/merkledag" + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + node "gx/ipfs/QmU7bFWQ793qmvNy7outdCaMfSDNk8uqhx4VNrxYj5fj5g/go-ipld-node" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" +) + +func NewDAGService(fs *Datastore, ds dag.DAGService) dag.DAGService { + return &dagService{fs, ds} +} + +type dagService struct { + fs *Datastore + dag.DAGService +} + +func GetLinks(dataObj *DataObj) ([]*node.Link, error) { + if !dataObj.Internal() { + return nil, nil + } + res, err := dag.DecodeProtobuf(dataObj.Data) + if err != nil { + return nil, err + } + return res.Links(), nil +} + +func (ds *dagService) GetLinks(ctx context.Context, c *cid.Cid) ([]*node.Link, error) { + dsKey := dshelp.CidToDsKey(c) + key := NewDbKey(dsKey.String(), "", -1, nil) + _, dataObj, err := ds.fs.GetDirect(key) + if err != nil { + return ds.DAGService.GetLinks(ctx, c) + } + return GetLinks(dataObj) +} + +func (ds *dagService) GetOfflineLinkService() dag.LinkService { + ds2 := ds.DAGService.GetOfflineLinkService() + if ds != ds2 { + return NewDAGService(ds.fs, ds.DAGService) + } else { + return ds2 + } +} diff --git a/filestore/util/clean.go b/filestore/util/clean.go new file mode 100644 index 00000000000..b357a0a8818 --- /dev/null +++ b/filestore/util/clean.go @@ -0,0 +1,224 @@ +package filestore_util + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "time" + + //bs "github.com/ipfs/go-ipfs/blocks/blockstore" + butil "github.com/ipfs/go-ipfs/blocks/blockstore/util" + cmds "github.com/ipfs/go-ipfs/commands" + "github.com/ipfs/go-ipfs/core" + . "github.com/ipfs/go-ipfs/filestore" + //"github.com/ipfs/go-ipfs/pin" + //fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + //dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + //cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" +) + +func Clean(req cmds.Request, node *core.IpfsNode, fs *Datastore, quiet bool, level int, what ...string) (io.Reader, error) { + //exclusiveMode := node.LocalMode() + stages := 0 // represented as a 3 digit octodecimal + // stage 0100: remove bad blocks + // 0020: remove incomplete nodes + // 0003: remove orphan nodes + to_remove := make([]bool, 100) + incompleteWhen := make([]string, 0) + for i := 0; i < len(what); i++ { + switch what[i] { + case "invalid": + what = append(what, "changed", "no-file") + case "full": + what = append(what, "invalid", "incomplete", "orphan") + case "changed": + stages |= 0100 + incompleteWhen = append(incompleteWhen, "changed") + to_remove[StatusFileChanged] = true + case "no-file": + stages |= 0100 + incompleteWhen = append(incompleteWhen, "no-file") + to_remove[StatusFileMissing] = true + case "error": + stages |= 0100 + incompleteWhen = append(incompleteWhen, "error") + to_remove[StatusFileError] = true + case "incomplete": + stages |= 0020 + to_remove[StatusIncomplete] = true + case "orphan": + stages |= 0003 + to_remove[StatusOrphan] = true + default: + return nil, errors.New("invalid arg: " + what[i]) + } + } + incompleteWhenStr := strings.Join(incompleteWhen, ",") + + rdr, wtr := io.Pipe() + var rmWtr io.Writer = wtr + if quiet { + rmWtr = ioutil.Discard + } + + snapshot, err := fs.GetSnapshot() + if err != nil { + return nil, err + } + + Logger.Debugf("Starting clean operation.") + + go func() { + // 123: verify-post-orphan required + // 12-: verify-full + // 1-3: verify-full required (verify-post-orphan would be incorrect) + // 1--: basic + // -23: verify-post-orphan required + // -2-: verify-full (cache optional) + // --3: verify-full required (verify-post-orphan would be incorrect) + // ---: nothing to do! + var ch <-chan ListRes + switch stages { + case 0100: + fmt.Fprintf(rmWtr, "performing verify --basic --level=%d\n", level) + ch, err = VerifyBasic(snapshot.Basic, &VerifyParams{ + Level: level, + Verbose: 1, + NoObjInfo: true, + }) + case 0120, 0103, 0003: + fmt.Fprintf(rmWtr, "performing verify --level=%d --incomplete-when=%s\n", + level, incompleteWhenStr) + ch, err = VerifyFull(node, snapshot, &VerifyParams{ + Level: level, + Verbose: 6, + IncompleteWhen: incompleteWhen, + NoObjInfo: true, + }) + case 0020: + fmt.Fprintf(rmWtr, "performing verify --skip-orphans --level=1\n") + ch, err = VerifyFull(node, snapshot, &VerifyParams{ + SkipOrphans: true, + Level: level, + Verbose: 6, + NoObjInfo: true, + }) + case 0123, 0023: + fmt.Fprintf(rmWtr, "performing verify --post-orphans --level=%d --incomplete-when=%s\n", + level, incompleteWhenStr) + ch, err = VerifyFull(node, snapshot, &VerifyParams{ + Level: level, + Verbose: 6, + IncompleteWhen: incompleteWhen, + PostOrphan: true, + NoObjInfo: true, + }) + default: + // programmer error + panic(fmt.Errorf("invalid stage string %d", stages)) + } + if err != nil { + wtr.CloseWithError(err) + return + } + + remover := NewFilestoreRemover(snapshot) + + ch2 := make(chan interface{}, 16) + go func() { + defer close(ch2) + for r := range ch { + if to_remove[r.Status] { + r2 := remover.Delete(KeyToKey(r.Key), nil) + if r2 != nil { + ch2 <- r2 + } + } + } + }() + err2 := butil.ProcRmOutput(ch2, rmWtr, wtr) + if err2 != nil { + wtr.CloseWithError(err2) + return + } + debugCleanRmDelay() + Logger.Debugf("Removing invalid blocks after clean. Online Mode.") + ch3 := remover.Finish(node.Blockstore, node.Pinning) + err2 = butil.ProcRmOutput(ch3, rmWtr, wtr) + if err2 != nil { + wtr.CloseWithError(err2) + return + } + wtr.Close() + }() + + return rdr, nil +} + +// func rmBlocks(mbs bs.MultiBlockstore, pins pin.Pinner, keys []*cid.Cid, snap Snapshot, fs *Datastore) <-chan interface{} { + +// // make the channel large enough to hold any result to avoid +// // blocking while holding the GCLock +// out := make(chan interface{}, len(keys)) + +// debugCleanRmDelay() + +// if snap.Defined() { +// Logger.Debugf("Removing invalid blocks after clean. Online Mode.") +// } else { +// Logger.Debugf("Removing invalid blocks after clean. Exclusive Mode.") +// } + +// prefix := fsrepo.FilestoreMount + +// go func() { +// defer close(out) + +// unlocker := mbs.GCLock() +// defer unlocker.Unlock() + +// stillOkay := butil.FilterPinned(mbs, pins, out, keys, prefix) + +// for _, k := range stillOkay { +// dbKey := NewDbKey(dshelp.CidToDsKey(k).String(), "", -1, k) +// var err error +// if snap.Defined() { +// origVal, err0 := snap.DB().Get(dbKey) +// if err0 != nil { +// out <- &butil.RemovedBlock{Hash: dbKey.Format(), Error: err.Error()} +// continue +// } +// dbKey = NewDbKey(dbKey.Hash, origVal.FilePath, int64(origVal.Offset), k) +// err = fs.DelDirect(dbKey, NotPinned) +// } else { +// // we have an exclusive lock +// err = fs.DB().Delete(dbKey.Bytes) +// } +// if err != nil { +// out <- &butil.RemovedBlock{Hash: dbKey.Format(), Error: err.Error()} +// } else { +// out <- &butil.RemovedBlock{Hash: dbKey.Format()} +// } +// } +// }() + +// return out +// } + +// this function is used for testing in order to test for race +// conditions +func debugCleanRmDelay() { + delayStr := os.Getenv("IPFS_FILESTORE_CLEAN_RM_DELAY") + if delayStr == "" { + return + } + delay, err := time.ParseDuration(delayStr) + if err != nil { + Logger.Warningf("Invalid value for IPFS_FILESTORE_CLEAN_RM_DELAY: %f", delay) + } + println("sleeping...") + time.Sleep(delay) +} diff --git a/filestore/util/common.go b/filestore/util/common.go new file mode 100644 index 00000000000..262086284ce --- /dev/null +++ b/filestore/util/common.go @@ -0,0 +1,400 @@ +package filestore_util + +import ( + "fmt" + "io" + "os" + "strings" + + . "github.com/ipfs/go-ipfs/filestore" + . "github.com/ipfs/go-ipfs/filestore/support" + + b "github.com/ipfs/go-ipfs/blocks/blockstore" + dag "github.com/ipfs/go-ipfs/merkledag" + node "gx/ipfs/QmU7bFWQ793qmvNy7outdCaMfSDNk8uqhx4VNrxYj5fj5g/go-ipld-node" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" + //cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + //"gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query" + //dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" +) + +type VerifyLevel int + +const ( + CheckExists VerifyLevel = iota + CheckFast + CheckIfChanged + CheckAlways +) + +func VerifyLevelFromNum(fs *Basic, level int) (VerifyLevel, error) { + switch level { + case 0, 1: + return CheckExists, nil + case 2, 3: + return CheckFast, nil + case 4, 5: + return CheckIfChanged, nil + case 6: + if fs.Verify() <= VerifyIfChanged { + return CheckIfChanged, nil + } else { + return CheckAlways, nil + } + case 7, 8, 9: + return CheckAlways, nil + default: + return -1, fmt.Errorf("verify level must be between 0-9: %d", level) + } +} + +const ( + //ShowOrphans = 1 + ShowSpecified = 2 + ShowTopLevel = 3 + //ShowFirstProblem = unimplemented + ShowProblemChildren = 5 + ShowChildren = 7 +) + +type Status int16 + +const ( + StatusNone Status = 0 // 00 = default + + CategoryOk Status = 0 + StatusOk Status = 1 // 01 = leaf node okay + StatusAllPartsOk Status = 2 // 02 = all children have "ok" status + StatusFound Status = 5 // 05 = Found key, but not in filestore + StatusOrphan Status = 8 + StatusAppended Status = 9 + + CategoryBlockErr Status = 10 // 1x means error with block + StatusFileError Status = 10 + StatusFileMissing Status = 11 + StatusFileChanged Status = 12 + StatusFileTouched Status = 13 + + CategoryNodeErr Status = 20 // 2x means error with non-block node + StatusProblem Status = 20 // 20 if some children exist but could not be read + StatusIncomplete Status = 21 + + CategoryOtherErr Status = 30 // 3x means error with database itself + StatusError Status = 30 + StatusCorrupt Status = 31 + StatusKeyNotFound Status = 32 + + CategoryUnchecked Status = 80 // 8x means unchecked + StatusUnchecked Status = 80 + StatusComplete Status = 82 // 82 = All parts found + + CategoryInternal Status = 90 + StatusMarked Status = 90 // 9x is for internal use +) + +func AnInternalError(status Status) bool { + return status == StatusError || status == StatusCorrupt +} + +func AnError(status Status) bool { + return Status(10) <= status && status < Status(80) +} + +func IsOk(status Status) bool { + return status == StatusOk || status == StatusAllPartsOk +} + +func Unchecked(status Status) bool { + return status == StatusUnchecked || status == StatusComplete +} + +func InternalNode(status Status) bool { + return status == StatusAllPartsOk || status == StatusIncomplete || + status == StatusProblem || status == StatusComplete +} + +func OfInterest(status Status) bool { + return !IsOk(status) && !Unchecked(status) +} + +func statusStr(status Status) string { + switch status { + case StatusNone: + return "" + case StatusOk, StatusAllPartsOk: + return "ok " + case StatusFound: + return "found " + case StatusAppended: + return "appended " + case StatusOrphan: + return "orphan " + case StatusFileError: + return "error " + case StatusFileMissing: + return "no-file " + case StatusFileChanged: + return "changed " + case StatusFileTouched: + return "touched " + case StatusIncomplete: + return "incomplete " + case StatusProblem: + return "problem " + case StatusError: + return "ERROR " + case StatusKeyNotFound: + return "missing " + case StatusCorrupt: + return "ERROR " + case StatusUnchecked: + return " " + case StatusComplete: + return "complete " + default: + return fmt.Sprintf("?%02d ", status) + } +} + +type ListRes struct { + Key Key + *DataObj + Status Status +} + +var EmptyListRes = ListRes{Key{"", "", -1}, nil, 0} + +func (r *ListRes) What() string { + if r.WholeFile() { + return "root" + } else { + return "leaf" + } +} + +func (r *ListRes) StatusStr() string { + str := statusStr(r.Status) + str = strings.TrimRight(str, " ") + if str == "" { + str = "unchecked" + } + return str +} + +func (r *ListRes) MHash() string { + return MHash(r.Key) +} + +func (r *ListRes) FormatHashOnly() string { + if r.Key.Hash == "" { + return "\n" + } else { + return fmt.Sprintf("%s%s\n", statusStr(r.Status), MHash(r.Key)) + } +} + +func (r *ListRes) FormatKeyOnly() string { + if r.Key.Hash == "" { + return "\n" + } else { + return fmt.Sprintf("%s%s\n", statusStr(r.Status), r.Key.Format()) + } +} + +func (r *ListRes) FormatDefault(fullKey bool) string { + if r.Key.Hash == "" { + return "\n" + } else if r.DataObj == nil { + return fmt.Sprintf("%s%s\n", statusStr(r.Status), r.Key.Format()) + } else { + return fmt.Sprintf("%s%s\n", statusStr(r.Status), r.DataObj.KeyStr(r.Key, fullKey)) + } +} + +func (r *ListRes) FormatWithType(fullKey bool) string { + if r.Key.Hash == "" { + return "\n" + } else if r.DataObj == nil { + return fmt.Sprintf("%s %s\n", statusStr(r.Status), r.Key.Format()) + } else { + return fmt.Sprintf("%s%s %s\n", statusStr(r.Status), r.TypeStr(), r.DataObj.KeyStr(r.Key, fullKey)) + } +} + +func (r *ListRes) FormatLong(fullKey bool) string { + if r.Key.Hash == "" { + return "\n" + } else if r.DataObj == nil { + return fmt.Sprintf("%s%49s %s\n", statusStr(r.Status), "", r.Key.Format()) + } else if r.NoBlockData() { + return fmt.Sprintf("%s%s %12d %30s %s\n", statusStr(r.Status), r.TypeStr(), r.Size, r.DateStr(), r.DataObj.KeyStr(r.Key, fullKey)) + } else { + return fmt.Sprintf("%s%s %12d %30s %s\n", statusStr(r.Status), r.TypeStr(), r.Size, "", r.DataObj.KeyStr(r.Key, fullKey)) + } +} + +func StrToFormatFun(str string, fullKey bool) (func(*ListRes) (string,error), error) { + switch str { + case "hash": + return func(r *ListRes) (string,error) { + return r.FormatHashOnly(), nil + }, nil + case "key": + return func(r *ListRes) (string,error) { + return r.FormatKeyOnly(), nil + }, nil + case "default", "": + return func(r *ListRes) (string,error) { + return r.FormatDefault(fullKey), nil + }, nil + case "w/type": + return func(r *ListRes) (string,error) { + return r.FormatWithType(fullKey), nil + }, nil + case "long": + return func(r *ListRes) (string,error) { + return r.FormatLong(fullKey), nil + }, nil + default: + return nil, fmt.Errorf("invalid format type: %s", str) + } +} + +func ListKeys(d *Basic) <-chan ListRes { + ch, _ := List(d, nil, true) + return ch +} + +type ListFilter func(*DataObj) bool + +func List(d *Basic, filter ListFilter, keysOnly bool) (<-chan ListRes, error) { + iter := ListIterator{d.DB().NewIterator(), filter} + + if keysOnly { + out := make(chan ListRes, 1024) + go func() { + defer close(out) + for iter.Next() { + out <- ListRes{Key: iter.Key().Key} + } + }() + return out, nil + } else { + out := make(chan ListRes, 128) + go func() { + defer close(out) + for iter.Next() { + res := ListRes{Key: iter.Key().Key} + res.DataObj, _ = iter.Value() + out <- res + } + }() + return out, nil + } +} + +var ListFilterAll ListFilter = nil + +func ListFilterWholeFile(r *DataObj) bool { return r.WholeFile() } + +func ListByKey(fs *Basic, ks []*DbKey) (<-chan ListRes, error) { + out := make(chan ListRes, 128) + + go func() { + defer close(out) + for _, k := range ks { + res, _ := fs.GetAll(k) + for _, kv := range res { + out <- ListRes{Key: kv.Key.Key, DataObj: kv.Val} + } + } + }() + return out, nil +} + +type ListIterator struct { + *Iterator + Filter ListFilter +} + +func (itr ListIterator) Next() bool { + for itr.Iterator.Next() { + if itr.Filter == nil { + return true + } + val, _ := itr.Value() + if val == nil { + // an error ... + return true + } + keep := itr.Filter(val) + if keep { + return true + } + // else continue to next value + } + return false +} + +func verify(d *Basic, key *DbKey, val *DataObj, level VerifyLevel) Status { + var err error + switch level { + case CheckExists: + return StatusUnchecked + case CheckFast: + err = VerifyFast(val) + case CheckIfChanged: + _, err = GetData(d.AsFull(), key, val, VerifyIfChanged) + case CheckAlways: + _, err = GetData(d.AsFull(), key, val, VerifyAlways) + default: + return StatusError + } + + if err == nil { + return StatusOk + } else if os.IsNotExist(err) { + return StatusFileMissing + } else if err == InvalidBlock || err == io.EOF || err == io.ErrUnexpectedEOF { + return StatusFileChanged + } else if err == TouchedBlock { + return StatusFileTouched + } else { + return StatusFileError + } +} + +func getNodes(key *DbKey, fs *Basic, bs b.Blockstore) ([]KeyVal, []*node.Link, Status) { + res, err := fs.GetAll(key) + if err == nil { + if res[0].Val.NoBlockData() { + return res, nil, StatusUnchecked + } else { + links, err := GetLinks(res[0].Val) + if err != nil { + Logger.Errorf("%s: %v", MHash(key), err) + return nil, nil, StatusCorrupt + } + return res[0:1], links, StatusOk + } + } + k, err2 := key.Cid() + if err2 != nil { + return nil, nil, StatusError + } + block, err2 := bs.Get(k) + if err == ds.ErrNotFound && err2 == b.ErrNotFound { + return nil, nil, StatusKeyNotFound + } else if err2 != nil { + Logger.Errorf("%s: %v", k, err) + Logger.Errorf("%s: %v", k, err2) + //panic(err2) + return nil, nil, StatusError + } + node, err := dag.DecodeProtobuf(block.RawData()) + if err != nil { + Logger.Errorf("%s: %v", k, err) + return nil, nil, StatusCorrupt + } + return nil, node.Links(), StatusFound +} diff --git a/filestore/util/misc.go b/filestore/util/misc.go new file mode 100644 index 00000000000..408ac0d51ba --- /dev/null +++ b/filestore/util/misc.go @@ -0,0 +1,61 @@ +package filestore_util + +import ( + "fmt" + "io" + + . "github.com/ipfs/go-ipfs/filestore" + + b "github.com/ipfs/go-ipfs/blocks/blockstore" + butil "github.com/ipfs/go-ipfs/blocks/blockstore/util" + "github.com/ipfs/go-ipfs/pin" + "github.com/ipfs/go-ipfs/repo/fsrepo" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + //k "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key" +) + +func Dups(wtr io.Writer, fs *Basic, bs b.MultiBlockstore, pins pin.Pinner, args ...string) error { + showPinned, showUnpinned := false, false + if len(args) == 0 { + showPinned, showUnpinned = true, true + } + for _, arg := range args { + switch arg { + case "pinned": + showPinned = true + case "unpinned": + showUnpinned = true + default: + return fmt.Errorf("invalid arg: %s", arg) + } + } + ls := ListKeys(fs) + dups := make([]*cid.Cid, 0) + for res := range ls { + c, err := res.Key.Cid() + if err != nil { + return err + } + if butil.AvailableElsewhere(bs, fsrepo.FilestoreMount, c) { + dups = append(dups, c) + } + } + if showPinned && showUnpinned { + for _, key := range dups { + fmt.Fprintf(wtr, "%s\n", key) + } + return nil + } + res, err := pins.CheckIfPinned(dups...) + if err != nil { + return err + } + for _, r := range res { + if showPinned && r.Pinned() { + fmt.Fprintf(wtr, "%s\n", r.Key) + } else if showUnpinned && !r.Pinned() { + fmt.Fprintf(wtr, "%s\n", r.Key) + } + } + return nil +} diff --git a/filestore/util/move.go b/filestore/util/move.go new file mode 100644 index 00000000000..08804df004b --- /dev/null +++ b/filestore/util/move.go @@ -0,0 +1,135 @@ +package filestore_util + +import ( + errs "errors" + "io" + "os" + "path/filepath" + + "github.com/ipfs/go-ipfs/core" + . "github.com/ipfs/go-ipfs/filestore" + "github.com/ipfs/go-ipfs/repo/fsrepo" + "github.com/ipfs/go-ipfs/unixfs" + + b "github.com/ipfs/go-ipfs/blocks/blockstore" + dag "github.com/ipfs/go-ipfs/merkledag" + //dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" +) + +// type fileNodes map[bk.Key]struct{} + +// func (m fileNodes) have(key bk.Key) bool { +// _, ok := m[key] +// return ok +//} + +//func (m fileNodes) add(key bk.Key) { +// m[key] = struct{}{} +//} + +// func extractFiles(key bk.Key, fs *Datastore, bs b.Blockservice, res *fileNodes) error { +// n, dataObj, status := getNode(key.DsKey(), key, fs, bs) +// if AnError(status) { +// return fmt.Errorf("Error when retrieving key: %s.", key) +// } +// if dataObj != nil { +// // already in filestore +// return nil +// } +// fsnode, err := unixfs.FromBytes(n.Data) +// if err != nil { +// return err +// } +// switch *fsnode.Type { +// case unixfs.TRaw: +// case unixfs.TFile: +// res.add(key) +// case unixfs.TDirectory: +// for _, link := range n.Links { +// err := extractFiles(bk.Key(link.Hash), fs, bs, res) +// if err != nil { +// return err +// } +// } +// default: +// } +// return nil +// } + +func ConvertToFile(node *core.IpfsNode, k *cid.Cid, path string) error { + config, _ := node.Repo.Config() + if !node.LocalMode() && (config == nil || !config.Filestore.APIServerSidePaths) { + return errs.New("Daemon is running and server side paths are not enabled.") + } + if !filepath.IsAbs(path) { + return errs.New("absolute path required") + } + wtr, err := os.Create(path) + if err != nil { + return err + } + fs, ok := node.Repo.DirectMount(fsrepo.FilestoreMount).(*Datastore) + if !ok { + return errs.New("Could not extract filestore.") + } + p := params{node.Blockstore, fs, path, wtr} + _, err = p.convertToFile(k, true, 0) + return err +} + +type params struct { + bs b.Blockstore + fs *Datastore + path string + out io.Writer +} + +func (p *params) convertToFile(k *cid.Cid, root bool, offset uint64) (uint64, error) { + block, err := p.bs.Get(k) + if err != nil { + return 0, err + } + altData, fsInfo, err := Reconstruct(block.RawData(), nil, 0) + if err != nil { + return 0, err + } + if fsInfo.Type != unixfs.TRaw && fsInfo.Type != unixfs.TFile { + return 0, errs.New("Not a file") + } + dataObj := &DataObj{ + FilePath: p.path, + Offset: offset, + Size: fsInfo.FileSize, + } + if root { + dataObj.Flags = WholeFile + } + if len(fsInfo.Data) > 0 { + _, err := p.out.Write(fsInfo.Data) + if err != nil { + return 0, err + } + dataObj.Flags |= NoBlockData + dataObj.Data = altData + return 0, errs.New("Unimplemeted") + //p.fs.Update(dshelp.CidToDsKey(k).Bytes(), nil, dataObj) + } else { + dataObj.Flags |= Internal + dataObj.Data = block.RawData() + return 0, errs.New("Unimplemeted") + //p.fs.Update(dshelp.CidToDsKey(k).Bytes(), nil, dataObj) + n, err := dag.DecodeProtobuf(block.RawData()) + if err != nil { + return 0, err + } + for _, link := range n.Links() { + size, err := p.convertToFile(link.Cid, false, offset) + if err != nil { + return 0, err + } + offset += size + } + } + return fsInfo.FileSize, nil +} diff --git a/filestore/util/remove.go b/filestore/util/remove.go new file mode 100644 index 00000000000..66c5d660075 --- /dev/null +++ b/filestore/util/remove.go @@ -0,0 +1,207 @@ +package filestore_util + +import ( + //"fmt" + //"io" + + bs "github.com/ipfs/go-ipfs/blocks/blockstore" + u "github.com/ipfs/go-ipfs/blocks/blockstore/util" + . "github.com/ipfs/go-ipfs/filestore" + . "github.com/ipfs/go-ipfs/filestore/support" + "github.com/ipfs/go-ipfs/pin" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" +) + +type FilestoreRemover struct { + ss Snapshot + tocheck []*cid.Cid + ReportFound bool + ReportNotFound bool + ReportAlreadyDeleted bool + Recursive bool + AllowNonRoots bool +} + +// Batch removal of filestore blocks works on a snapshot of the +// database and is done in two passes. + +// In the first pass a DataObj is deleted if it can be done so without +// requiring a pin check. If a pin check is required than the hash is +// appened to the tocheck for the pin check. +// +// By definition if a hash in in tocheck there is only one DataObj for +// that hash left at the time the snapshot was taken. +// +// In the second pass the pincheck is done and any unpinned hashes are +// deleted. In the case that there is multiple DataObjs in the +// snapshot all but one will have already been removed; however we +// can't easily tell which one so just re-delete all the keys present +// in the snapshot. + +func NewFilestoreRemover(ss Snapshot) *FilestoreRemover { + return &FilestoreRemover{ss: ss, ReportFound: true, ReportNotFound: true} +} + +func (r *FilestoreRemover) Delete(key *DbKey, dataObj *DataObj) *u.RemovedBlock { + var err error + if dataObj == nil { + _, dataObj, err = r.ss.GetDirect(key) + if err == ds.ErrNotFound { + if r.ReportNotFound { + return &u.RemovedBlock{Hash: key.Format(), Error: err.Error()} + } else { + return nil + } + } else if err != nil { + return &u.RemovedBlock{Hash: key.Format(), Error: err.Error()} + } + } + fullKey := key.MakeFull(dataObj) + err = r.ss.AsFull().DelSingle(fullKey, MaybePinned) + if err == ds.ErrNotFound { + if r.ReportAlreadyDeleted { + return &u.RemovedBlock{Hash: fullKey.Format(), Error: "already deleted"} + } else { + return nil + } + } else if err == RequirePinCheck { + c, _ := fullKey.Cid() + r.tocheck = append(r.tocheck, c) + return nil + } else if err != nil { + return &u.RemovedBlock{Hash: fullKey.Format(), Error: err.Error()} + } else { + if r.ReportFound { + return &u.RemovedBlock{Hash: fullKey.Format()} + } else { + return nil + } + } +} + +func (r *FilestoreRemover) DeleteAll(key *DbKey, out chan<- interface{}) { + kvs, err := r.ss.GetAll(key) + if err == ds.ErrNotFound { + if r.ReportNotFound { + out <- &u.RemovedBlock{Hash: key.Format(), Error: err.Error()} + } + } else if err != nil { + out <- &u.RemovedBlock{Hash: key.Format(), Error: err.Error()} + } + requireRoot := !r.AllowNonRoots && key.FilePath == "" + for _, kv := range kvs { + fullKey := kv.Key.MakeFull(kv.Val) + if requireRoot && !kv.Val.WholeFile() { + out <- &u.RemovedBlock{Hash: fullKey.Format(), Error: "key is not a root and no file path was specified"} + continue + } + if r.Recursive { + r.DeleteRec(kv.Key, kv.Val, out) + } else { + res := r.Delete(kv.Key, kv.Val) + if res != nil { + out <- res + } + } + } +} + +func (r *FilestoreRemover) DeleteRec(key *DbKey, dataObj *DataObj, out chan<- interface{}) { + res := r.Delete(key, dataObj) + if res != nil { + out <- res + } + filePath := dataObj.FilePath + links, err := GetLinks(dataObj) + if err != nil { + out <- &u.RemovedBlock{Hash: key.Format(), Error: err.Error()} + return + } + for _, link := range links { + // only delete entries that have a matching FilePath + k := NewDbKey(dshelp.CidToDsKey(link.Cid).String(), filePath, -1, link.Cid) + kvs, err := r.ss.GetAll(k) + if err == ds.ErrNotFound { + if r.ReportNotFound { + out <- &u.RemovedBlock{Hash: k.Format(), Error: err.Error()} + } + } else if err != nil { + out <- &u.RemovedBlock{Hash: k.Format(), Error: err.Error()} + } + for _, kv := range kvs { + r.DeleteRec(kv.Key, kv.Val, out) + } + } +} + +func (r *FilestoreRemover) Finish(mbs bs.MultiBlockstore, pins pin.Pinner) <-chan interface{} { + // make the channel large enough to hold any result to avoid + // blocking while holding the GCLock + out := make(chan interface{}, len(r.tocheck)) + prefix := fsrepo.FilestoreMount + + if len(r.tocheck) == 0 { + close(out) + return out + } + + go func() { + defer close(out) + + unlocker := mbs.GCLock() + defer unlocker.Unlock() + + stillOkay := u.FilterPinned(mbs, pins, out, r.tocheck, prefix) + + for _, c := range stillOkay { + k := CidToKey(c) + todel, err := r.ss.GetAll(k) + if err != nil { + out <- &u.RemovedBlock{Hash: k.Format(), Error: err.Error()} + } + for _, kv := range todel { + dataObj := kv.Val + dbKey := k.MakeFull(dataObj) + err = r.ss.AsFull().DelDirect(dbKey, NotPinned) + if err == ds.ErrNotFound { + if r.ReportAlreadyDeleted { + out <- &u.RemovedBlock{Hash: dbKey.Format(), Error: "already deleted"} + } + } else if err != nil { + out <- &u.RemovedBlock{Hash: dbKey.Format(), Error: err.Error()} + } else { + if r.ReportFound { + out <- &u.RemovedBlock{Hash: dbKey.Format()} + } + } + } + } + }() + + return out +} + +// func RmBlocks(fs *Datastore, mbs bs.MultiBlockstore, pins pin.Pinner, out chan<- interface{}, keys []*DbKey) error { +// ss,err := fs.GetSnapshot() +// if err != nil { +// return err +// } +// r := NewFilestoreRemover(ss) +// go func() { +// defer close(out) +// for _, k := range keys { +// res := r.Delete(k) +// if res != nil { +// out <- res +// } +// } +// out2 := r.Finish(mbs, pins) +// for res := range out2 { +// out <- res +// } +// }() +// return nil +// } diff --git a/filestore/util/verify.go b/filestore/util/verify.go new file mode 100644 index 00000000000..5fba8d6d65a --- /dev/null +++ b/filestore/util/verify.go @@ -0,0 +1,556 @@ +package filestore_util + +import ( + "errors" + "fmt" + "os" + //"sync" + //"strings" + + b "github.com/ipfs/go-ipfs/blocks/blockstore" + "github.com/ipfs/go-ipfs/core" + . "github.com/ipfs/go-ipfs/filestore" + . "github.com/ipfs/go-ipfs/filestore/support" + dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help" + node "gx/ipfs/QmU7bFWQ793qmvNy7outdCaMfSDNk8uqhx4VNrxYj5fj5g/go-ipld-node" + //cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" +) + +type VerifyParams struct { + Filter ListFilter + Level int + Verbose int + NoObjInfo bool + SkipOrphans bool + PostOrphan bool + IncompleteWhen []string +} + +func CheckParamsBasic(fs *Basic, params *VerifyParams) (VerifyLevel, int, error) { + level, err := VerifyLevelFromNum(fs, params.Level) + if err != nil { + return 0, 0, err + } + verbose := params.Verbose + if verbose < 0 || verbose > 9 { + return 0, 0, errors.New("verbose must be between 0-9") + } + return level, verbose, nil +} + +func ParseIncompleteWhen(args []string) ([]bool, error) { + ret := make([]bool, 100) + ret[StatusKeyNotFound] = true + ret[StatusIncomplete] = true + for _, arg := range args { + switch arg { + case "changed": + ret[StatusFileChanged] = true + case "no-file": + ret[StatusFileMissing] = true + case "error": + ret[StatusFileError] = true + default: + return nil, fmt.Errorf("IncompleteWhen: Expect one of: changed, no-file, error. Got: %s", arg) + } + } + return ret, nil +} + +type reporter struct { + ch chan ListRes + noObjInfo bool +} + +func (out *reporter) send(res ListRes) { + if out.noObjInfo { + res.DataObj = nil + } + out.ch <- res +} + +func (out *reporter) close() { + close(out.ch) +} + +func VerifyBasic(fs *Basic, params *VerifyParams) (<-chan ListRes, error) { + iter := ListIterator{Iterator: fs.DB().NewIterator()} + if params.Filter == nil { + iter.Filter = func(r *DataObj) bool { return r.NoBlockData() } + } else { + iter.Filter = func(r *DataObj) bool { return r.NoBlockData() && params.Filter(r) } + } + verifyLevel, verbose, err := CheckParamsBasic(fs, params) + if err != nil { + return nil, err + } + out := reporter{make(chan ListRes, 16), params.NoObjInfo} + go func() { + defer out.close() + for iter.Next() { + key := iter.Key() + dataObj, err := iter.Value() + if err != nil { + out.send(ListRes{key.Key, nil, StatusCorrupt}) + } + status := verify(fs, key, dataObj, verifyLevel) + if verbose >= ShowTopLevel || OfInterest(status) { + out.send(ListRes{key.Key, dataObj, status}) + } + } + }() + return out.ch, nil +} + +func VerifyKeys(ks []*DbKey, node *core.IpfsNode, fs *Basic, params *VerifyParams) (<-chan ListRes, error) { + out := reporter{make(chan ListRes, 16), params.NoObjInfo} + verifyLevel, verbose, err := CheckParamsBasic(fs, params) + if err != nil { + return nil, err + } + go func() { + defer out.close() + for _, k := range ks { + //if key == "" { + // continue + //} + res := verifyKey(k, fs, node.Blockstore, verifyLevel) + if verbose >= ShowSpecified || OfInterest(res.Status) { + out.send(res) + } + } + }() + return out.ch, nil +} + +func verifyKey(dsKey *DbKey, fs *Basic, bs b.Blockstore, verifyLevel VerifyLevel) ListRes { + _, dataObj, err := fs.GetDirect(dsKey) + if err == nil && dataObj.NoBlockData() { + res := ListRes{dsKey.Key, dataObj, 0} + res.Status = verify(fs, dsKey, dataObj, verifyLevel) + return res + } else if err == nil { + return ListRes{dsKey.Key, dataObj, StatusUnchecked} + } + c, _ := dsKey.Cid() + found, _ := bs.Has(c) + if found { + return ListRes{dsKey.Key, nil, StatusFound} + } else if err == ds.ErrNotFound && !found { + return ListRes{dsKey.Key, nil, StatusKeyNotFound} + } else { + Logger.Errorf("%s: verifyKey: %v", dsKey.Format(), err) + return ListRes{dsKey.Key, nil, StatusError} + } +} + +func VerifyFull(node *core.IpfsNode, fs Snapshot, params *VerifyParams) (<-chan ListRes, error) { + verifyLevel, verbose, err := CheckParamsBasic(fs.Basic, params) + if err != nil { + return nil, err + } + skipOrphans := params.SkipOrphans + postOrphan := params.PostOrphan + if skipOrphans && postOrphan { + return nil, fmt.Errorf("cannot specify both skip-orphans and post-orphan") + } + if params.Filter != nil { + skipOrphans = true + postOrphan = false + } + p := verifyParams{ + out: reporter{make(chan ListRes, 16), params.NoObjInfo}, + node: node, + fs: fs.Basic, + verifyLevel: verifyLevel, + verboseLevel: verbose, + } + p.incompleteWhen, err = ParseIncompleteWhen(params.IncompleteWhen) + if err != nil { + return nil, err + } + iter := ListIterator{fs.DB().NewIterator(), params.Filter} + go func() { + defer p.out.close() + switch { + case skipOrphans: + p.verifyRecursive(iter) + case postOrphan: + p.verifyPostOrphan(iter) + default: + p.verifyFull(iter) + } + }() + return p.out.ch, nil +} + +func VerifyKeysFull(ks []*DbKey, node *core.IpfsNode, fs *Basic, params *VerifyParams) (<-chan ListRes, error) { + verifyLevel, verbose, err := CheckParamsBasic(fs, params) + if err != nil { + return nil, err + } + p := verifyParams{ + out: reporter{make(chan ListRes, 16), params.NoObjInfo}, + node: node, + fs: fs, + verifyLevel: verifyLevel, + verboseLevel: verbose, + } + p.incompleteWhen, err = ParseIncompleteWhen(params.IncompleteWhen) + if err != nil { + return nil, err + } + go func() { + defer p.out.close() + p.verifyKeys(ks) + }() + return p.out.ch, nil +} + +// type VerifyType int + +// const ( +// Recursive VerifyType = iota +// Full +// PostOrphan +// ) + +type Hash string + +type seen struct { + status Status + reachable bool +} + +type verifyParams struct { + out reporter + node *core.IpfsNode + fs *Basic + verifyLevel VerifyLevel + verboseLevel int // see help text for meaning + seen map[string]seen + roots []string + incompleteWhen []bool +} + +func (p *verifyParams) getStatus(key string) seen { + if p.seen == nil { + return seen{0, false} + } else { + return p.seen[key] + } +} + +func (p *verifyParams) setStatus(key *DbKey, val *DataObj, status Status, reachable bool) { + if p.seen != nil { + val, ok := p.seen[key.Hash] + if status > 0 && !ok { + p.seen[key.Hash] = seen{status, reachable} + } else { + if status > 0 {val.status = status} + if reachable {val.reachable = true} + p.seen[key.Hash] = val + } + } + if p.roots != nil && val != nil && val.WholeFile() { + p.roots = append(p.roots, key.Hash) + } +} + +func (p *verifyParams) verifyKeys(ks []*DbKey) { + for _, dsKey := range ks { + //if key == "" { + // continue + //} + res, children, r := p.get(dsKey) + if res == nil || AnError(r) { + /* nothing to do */ + } else if res[0].Val.Internal() { + kv := res[0] + r = p.verifyNode(children) + p.verifyPostTopLevel(kv.Key, kv.Val, r) + return + } + for _, kv := range res { + r = p.verifyLeaf(kv.Key, kv.Val) + p.verifyPostTopLevel(kv.Key, kv.Val, r) + } + } +} + +func (p *verifyParams) verifyPostTopLevel(dsKey *DbKey, dataObj *DataObj, r Status) { + res := ListRes{dsKey.Key, dataObj, r} + res.Status = p.checkIfAppended(res) + if p.verboseLevel >= ShowSpecified || OfInterest(res.Status) { + p.out.send(res) + p.out.ch <- EmptyListRes + } +} + +func (p *verifyParams) verifyRecursive(iter ListIterator) { + p.verifyTopLevel(iter) +} + +func (p *verifyParams) verifyFull(iter ListIterator) error { + p.seen = make(map[string]seen) + + err := p.verifyTopLevel(iter) + // An error indicates an internal error that might mark some nodes + // incorrectly as orphans, so exit early + if err != nil { + return InternalError + } + + p.checkOrphans() + + return nil +} + +func (p *verifyParams) verifyPostOrphan(iter ListIterator) error { + p.seen = make(map[string]seen) + p.roots = make([]string, 0) + + reportErr := p.verifyTopLevel(iter) + + err := p.markReachable(p.roots) + + if reportErr != nil || err != nil { + return InternalError + } + + p.outputFutureOrphans() + + p.checkOrphans() + + return nil +} + +var InternalError = errors.New("database corrupt or related") + +func (p *verifyParams) verifyTopLevel(iter ListIterator) error { + unsafeToCont := false + for iter.Next() { + key := iter.Key() + r := StatusUnchecked + val, err := iter.Value() + if err != nil { + r = StatusCorrupt + } + if AnError(r) { + p.reportTopLevel(key, val, r) + } else if val.Internal() && val.WholeFile() { + children, err := GetLinks(val) + if err != nil { + r = StatusCorrupt + } else { + r = p.verifyNode(children) + } + p.reportTopLevel(key, val, r) + p.setStatus(key, val, r, false) + } else if val.WholeFile() { + r = p.verifyLeaf(key, val) + p.reportTopLevel(key, val, r) + // mark the node as seen, but do not cache the status as + // that status might be incomplete + p.setStatus(key, val, StatusUnchecked, false) + } else { + // FIXME: Is this doing anything useful? + p.setStatus(key, val, 0, false) + continue + } + if AnInternalError(r) { + unsafeToCont = true + } + } + if unsafeToCont { + return InternalError + } else { + return nil + } +} + +func (p *verifyParams) reportTopLevel(key *DbKey, val *DataObj, status Status) { + res := ListRes{key.Key, val, status} + res.Status = p.checkIfAppended(res) + if p.verboseLevel >= ShowTopLevel || (p.verboseLevel >= 0 && OfInterest(res.Status)) { + p.out.send(res) + p.out.ch <- EmptyListRes + } +} + +func (p *verifyParams) checkOrphans() { + for k, v := range p.seen { + if v.reachable { + continue + } + p.outputOrphans(k, v.status) + } +} + +func (p *verifyParams) checkIfAppended(res ListRes) Status { + //println("checkIfAppened:", res.FormatDefault()) + if p.verifyLevel <= CheckExists || p.verboseLevel < 0 || + !IsOk(res.Status) || !res.WholeFile() || res.FilePath == "" { + //println("checkIfAppened no go", res.FormatDefault()) + return res.Status + } + //println("checkIfAppened no checking", res.FormatDefault()) + info, err := os.Stat(res.FilePath) + if err != nil { + Logger.Warningf("%s: checkIfAppended: %v", res.MHash(), err) + return res.Status + } + if uint64(info.Size()) > res.Size { + return StatusAppended + } + return res.Status +} + +var depth = 0 + +func (p *verifyParams) markReachable(keys []string) error { + depth += 1 + for _, hash := range keys { + v := p.seen[hash] + r := v.status + if r == StatusMarked { + continue + } + if AnInternalError(r) { // not stricly necessary, but lets be extra safe + return InternalError + } + //println("status", HashToKey(hash).Format(), r) + if InternalNode(r) && r != StatusIncomplete { + key := HashToKey(hash) + _, val, err := p.fs.GetDirect(key) + if err != nil { + //println("um an error") + return err + } + links, err := GetLinks(val) + children := make([]string, 0, len(links)) + for _, link := range links { + children = append(children, dshelp.CidToDsKey(link.Cid).String()) + } + //println("recurse", depth, HashToKey(hash).Format(), "count", len(children)) + p.markReachable(children) + } + //println("seen", depth, HashToKey(hash).Format()) + v.status = StatusMarked + p.seen[hash] = v + } + depth -= 1 + return nil +} + +func (p *verifyParams) outputFutureOrphans() { + for hash, v := range p.seen { + if v.status == StatusMarked || v.status == StatusNone { + continue + } + p.outputOrphans(hash, v.status) + } +} + +func (p *verifyParams) outputOrphans(hashStr string, status Status) { + hash := HashToKey(hashStr) + kvs, err := p.fs.GetAll(hash) + if err != nil { + Logger.Errorf("%s: verify: %v", MHash(hash), err) + p.out.send(ListRes{hash.Key, nil, StatusError}) + } + for _, kv := range kvs { + if kv.Val.WholeFile() { + continue + } + if status == StatusNone && kv.Val.NoBlockData() { + r := p.verifyLeaf(kv.Key, kv.Val) + if AnError(r) { + p.out.send(ListRes{kv.Key.Key, kv.Val, r}) + } + } + p.out.send(ListRes{kv.Key.Key, kv.Val, StatusOrphan}) + } +} + +func (p *verifyParams) verifyNode(links []*node.Link) Status { + finalStatus := StatusComplete + for _, link := range links { + hash := CidToKey(link.Cid) + v := p.getStatus(hash.Hash) + var dataObj *DataObj + if v.status == 0 { + objs, children, r := p.get(hash) + if objs != nil { + dataObj = objs[0].Val + } + if AnError(r) { + p.reportNodeStatus(hash, dataObj, r) + } else if len(children) > 0 { + r = p.verifyNode(children) + p.reportNodeStatus(hash, dataObj, r) + } else if objs != nil { + r = StatusNone + for _, kv := range objs { + r0 := p.verifyLeaf(kv.Key, kv.Val) + p.reportNodeStatus(kv.Key, kv.Val, r0) + if p.rank(r0) < p.rank(r) { + r = r0 + } + } + } + v.status = r + } + p.setStatus(hash, dataObj, v.status, true) + if AnInternalError(v.status) { + return StatusError + } else if p.incompleteWhen[v.status] { + finalStatus = StatusIncomplete + } else if !IsOk(v.status) && !Unchecked(v.status) { + finalStatus = StatusProblem + } + } + if finalStatus == StatusComplete && p.verifyLevel > CheckExists { + finalStatus = StatusAllPartsOk + } + return finalStatus +} + +func (p *verifyParams) reportNodeStatus(key *DbKey, val *DataObj, status Status) { + if p.verboseLevel >= ShowChildren || (p.verboseLevel >= ShowProblemChildren && OfInterest(status)) { + p.out.send(ListRes{key.Key, val, status}) + } +} + +// determine the rank of the status indicator if multiple entries have +// the same hash and differnt status, the one with the lowest rank +// will be used +func (p *verifyParams) rank(r Status) int { + category := r - r%10 + switch { + case r == 0: + return 999 + case category == CategoryOk: + return int(r) + case category == CategoryUnchecked: + return 100 + int(r) + case category == CategoryBlockErr && !p.incompleteWhen[r]: + return 200 + int(r) + case category == CategoryBlockErr && p.incompleteWhen[r]: + return 400 + int(r) + case category == CategoryOtherErr: + return 500 + int(r) + default: + // should not really happen + return 600 + int(r) + } +} + +func (p *verifyParams) verifyLeaf(key *DbKey, dataObj *DataObj) Status { + return verify(p.fs, key, dataObj, p.verifyLevel) +} + +func (p *verifyParams) get(k *DbKey) ([]KeyVal, []*node.Link, Status) { + return getNodes(k, p.fs, p.node.Blockstore) +} diff --git a/package.json b/package.json index b26aa3a5757..0ddccdbf4fb 100644 --- a/package.json +++ b/package.json @@ -276,6 +276,12 @@ "name": "go-ipld-node", "version": "0.3.2" }, + { + "author": "kevina", + "hash": "QmRpAnJ1Mvd2wCtwoFevW8pbLTivUqmFxynptG6uvp1jzC", + "name": "safepath", + "version": "0.0.1" + }, { "author": "whyrusleeping", "hash": "QmRcAVqrbY5wryx7hfNLtiUZbCcstzaJL7YJFBboitcqWF", @@ -296,4 +302,3 @@ "name": "go-ipfs", "version": "0.4.5-dev" } - diff --git a/pin/gc/gc.go b/pin/gc/gc.go index d2607bdbef5..286f4e0f2d4 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -22,7 +22,7 @@ var log = logging.Logger("gc") // // The routine then iterates over every block in the blockstore and // deletes any block that is not found in the marked set. -func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan *cid.Cid, error) { +func GC(ctx context.Context, bs bstore.MultiBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan *cid.Cid, error) { unlocker := bs.GCLock() ls = ls.GetOfflineLinkService() @@ -32,7 +32,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin. return nil, err } - keychan, err := bs.AllKeysChan(ctx) + // only delete blocks in the first (cache) mount + keychan, err := bs.FirstMount().AllKeysChan(ctx) if err != nil { return nil, err } diff --git a/repo/config/config.go b/repo/config/config.go index 898cf56a472..c9b54fefecb 100644 --- a/repo/config/config.go +++ b/repo/config/config.go @@ -19,6 +19,7 @@ var log = logging.Logger("config") type Config struct { Identity Identity // local node's peer identity Datastore Datastore // local node's storage + Filestore Filestore // local node's filestore Addresses Addresses // local node's addresses Mounts Mounts // local node's mount points Discovery Discovery // local node's discovery mechanisms diff --git a/repo/config/datastore.go b/repo/config/datastore.go index 2b861a113cf..c7bb9885108 100644 --- a/repo/config/datastore.go +++ b/repo/config/datastore.go @@ -40,3 +40,9 @@ type S3Datastore struct { func DataStorePath(configroot string) (string, error) { return Path(configroot, DefaultDataStoreDirectory) } + +type Filestore struct { + Verify string // one of "always", "ifchanged", "never" + APIServerSidePaths bool + NoDBCompression bool +} diff --git a/repo/fsrepo/defaultds.go b/repo/fsrepo/defaultds.go index ed8fbafe702..639d477e655 100644 --- a/repo/fsrepo/defaultds.go +++ b/repo/fsrepo/defaultds.go @@ -2,13 +2,18 @@ package fsrepo import ( "fmt" + "os" "path" + "strings" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" "github.com/ipfs/go-ipfs/thirdparty/dir" + + filestore "github.com/ipfs/go-ipfs/filestore" "gx/ipfs/QmU4VzzKNLJXJ72SedXBQKyf5Jo8W89iWpbWQjHn9qef8N/go-ds-flatfs" levelds "gx/ipfs/QmUHmMGmcwCrjHQHcYhBnqGCSWs5pBSMbGZmfwavETR1gg/go-ds-leveldb" + //multi "github.com/ipfs/go-ipfs/repo/multi" ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt" ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" mount "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/syncmount" @@ -18,9 +23,17 @@ import ( const ( leveldbDirectory = "datastore" flatfsDirectory = "blocks" + fileStoreDir = "filestore-db" + fileStoreDataDir = "filestore-data" ) -func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) { +const ( + RootMount = "/" + CacheMount = "/blocks" // needs to be the same as blockstore.DefaultPrefix + FilestoreMount = "/filestore" +) + +func openDefaultDatastore(r *FSRepo) (repo.Datastore, []Mount, error) { leveldbPath := path.Join(r.path, leveldbDirectory) // save leveldb reference so it can be neatly closed afterward @@ -28,7 +41,7 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) { Compression: ldbopts.NoCompression, }) if err != nil { - return nil, fmt.Errorf("unable to open leveldb datastore: %v", err) + return nil, nil, fmt.Errorf("unable to open leveldb datastore: %v", err) } syncfs := !r.config.Datastore.NoSync @@ -36,7 +49,7 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) { // by the Qm prefix. Leaving us with 9 bits, or 512 way sharding blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 5, syncfs) if err != nil { - return nil, fmt.Errorf("unable to open flatfs datastore: %v", err) + return nil, nil, fmt.Errorf("unable to open flatfs datastore: %v", err) } // Add our PeerID to metrics paths to keep them unique @@ -51,18 +64,37 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) { prefix := "fsrepo." + id + ".datastore." metricsBlocks := measure.New(prefix+"blocks", blocksDS) metricsLevelDB := measure.New(prefix+"leveldb", leveldbDS) - mountDS := mount.New([]mount.Mount{ - { - Prefix: ds.NewKey("/blocks"), - Datastore: metricsBlocks, - }, - { - Prefix: ds.NewKey("/"), - Datastore: metricsLevelDB, - }, + + var mounts []mount.Mount + var directMounts []Mount + + mounts = append(mounts, mount.Mount{ + Prefix: ds.NewKey(CacheMount), + Datastore: metricsBlocks, }) + directMounts = append(directMounts, Mount{CacheMount, blocksDS}) + + fileStore, err := r.newFilestore() + if err != nil { + return nil, nil, err + } + if fileStore != nil { + mounts = append(mounts, mount.Mount{ + Prefix: ds.NewKey(FilestoreMount), + Datastore: fileStore, + }) + directMounts = append(directMounts, Mount{FilestoreMount, fileStore}) + } - return mountDS, nil + mounts = append(mounts, mount.Mount{ + Prefix: ds.NewKey(RootMount), + Datastore: metricsLevelDB, + }) + directMounts = append(directMounts, Mount{RootMount, leveldbDS}) + + mountDS := mount.New(mounts) + + return mountDS, directMounts, nil } func initDefaultDatastore(repoPath string, conf *config.Config) error { @@ -79,3 +111,29 @@ func initDefaultDatastore(repoPath string, conf *config.Config) error { } return nil } + +func InitFilestore(repoPath string) error { + fileStorePath := path.Join(repoPath, fileStoreDir) + return filestore.Init(fileStorePath) +} + +// will return nil, nil if the filestore is not enabled +func (r *FSRepo) newFilestore() (*filestore.Datastore, error) { + fileStorePath := path.Join(r.path, fileStoreDir) + if _, err := os.Stat(fileStorePath); os.IsNotExist(err) { + return nil, nil + } + verify := filestore.VerifyAlways + switch strings.ToLower(r.config.Filestore.Verify) { + case "never": + verify = filestore.VerifyNever + case "ifchanged", "if changed": + verify = filestore.VerifyIfChanged + case "", "always": + verify = filestore.VerifyAlways + default: + return nil, fmt.Errorf("invalid value for Filestore.Verify: %s", r.config.Filestore.Verify) + } + println(verify) + return filestore.New(fileStorePath, verify, r.config.Filestore.NoDBCompression) +} diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 03ac313e63b..847ee7777c6 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -20,6 +20,7 @@ import ( dir "github.com/ipfs/go-ipfs/thirdparty/dir" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" util "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" "gx/ipfs/QmeqtHtxGfcsfXiou7wqHJARWPKUTUcPdtSfSYYHp48dtQ/go-ds-measure" ) @@ -93,6 +94,12 @@ type FSRepo struct { lockfile io.Closer config *config.Config ds repo.Datastore + mounts []Mount +} + +type Mount struct { + prefix string + dstore ds.Datastore } var _ repo.Repo = (*FSRepo)(nil) @@ -331,11 +338,12 @@ func (r *FSRepo) openConfig() error { func (r *FSRepo) openDatastore() error { switch r.config.Datastore.Type { case "default", "leveldb", "": - d, err := openDefaultDatastore(r) + d, m, err := openDefaultDatastore(r) if err != nil { return err } r.ds = d + r.mounts = m default: return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type) } @@ -557,6 +565,27 @@ func (r *FSRepo) Datastore() repo.Datastore { return d } +func (r *FSRepo) DirectMount(prefix string) ds.Datastore { + packageLock.Lock() + defer packageLock.Unlock() + for _, m := range r.mounts { + if prefix == m.prefix { + return m.dstore + } + } + return nil +} + +func (r *FSRepo) Mounts() []string { + packageLock.Lock() + mounts := make([]string, 0, len(r.mounts)) + for _, m := range r.mounts { + mounts = append(mounts, m.prefix) + } + packageLock.Unlock() + return mounts +} + // GetStorageUsage computes the storage space taken by the repo in bytes func (r *FSRepo) GetStorageUsage() (uint64, error) { pth, err := config.PathRoot() diff --git a/repo/mock.go b/repo/mock.go index 8190a0bda1b..f68e078cfbc 100644 --- a/repo/mock.go +++ b/repo/mock.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/ipfs/go-ipfs/repo/config" + ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore" ) var errTODO = errors.New("TODO: mock repo") @@ -33,6 +34,18 @@ func (m *Mock) GetConfigKey(key string) (interface{}, error) { func (m *Mock) Datastore() Datastore { return m.D } +func (m *Mock) DirectMount(prefix string) ds.Datastore { + if prefix == "/" { + return m.D + } else { + return nil + } +} + +func (m *Mock) Mounts() []string { + return []string{"/"} +} + func (m *Mock) GetStorageUsage() (uint64, error) { return 0, nil } func (m *Mock) Close() error { return errTODO } diff --git a/repo/repo.go b/repo/repo.go index d95af0446dd..633ff57114b 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -22,6 +22,14 @@ type Repo interface { Datastore() Datastore GetStorageUsage() (uint64, error) + // DirectMount provides direct access to a datastore mounted + // under prefix in order to perform low-level operations. The + // datastore returned is guaranteed not be a proxy (such as a + // go-datastore/measure) normal operations should go through + // Datastore() + DirectMount(prefix string) ds.Datastore + Mounts() []string + // SetAPIAddr sets the API address in the repo. SetAPIAddr(addr string) error diff --git a/test/sharness/lib/test-filestore-lib.sh b/test/sharness/lib/test-filestore-lib.sh new file mode 100644 index 00000000000..2b50b0cb0be --- /dev/null +++ b/test/sharness/lib/test-filestore-lib.sh @@ -0,0 +1,439 @@ +client_err() { + printf "$@\n\nUse 'ipfs add --help' for information about this command\n" +} + + +test_enable_filestore() { + test_expect_success "enable filestore" ' + ipfs filestore enable + ' +} + +test_add_cat_file() { + cmd=$1 + dir=$2 + HASH=$3 # QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH + + test_expect_success "ipfs $cmd succeeds" ' + echo "Hello Worlds!" >mountdir/hello.txt && + ipfs $cmd "$dir"/mountdir/hello.txt >actual + ' + + test_expect_success "ipfs $cmd output looks good" ' + echo "added $HASH "$dir"/mountdir/hello.txt" >expected && + test_cmp expected actual + ' + + test_expect_success "ipfs cat succeeds" ' + ipfs cat "$HASH" >actual + ' + + test_expect_success "ipfs cat output looks good" ' + echo "Hello Worlds!" >expected && + test_cmp expected actual + ' +} + +test_add_empty_file() { + cmd=$1 + dir=$2 + + EMPTY_HASH="QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH" + + test_expect_success "ipfs $cmd on empty file succeeds" ' + ipfs block rm -f $EMPTY_HASH && + cat /dev/null >mountdir/empty.txt && + ipfs $cmd "$dir"/mountdir/empty.txt >actual + ' + + test_expect_success "ipfs $cmd on empty file output looks good" ' + echo "added $EMPTY_HASH "$dir"/mountdir/empty.txt" >expected && + test_cmp expected actual + ' + + test_expect_success "ipfs cat on empty file succeeds" ' + ipfs cat "$EMPTY_HASH" >actual + ' + + test_expect_success "ipfs cat on empty file output looks good" ' + cat /dev/null >expected && + test_cmp expected actual + ' +} + +test_post_add() { + cmd=$1 + dir=$2 + + test_expect_success "fail after file move" ' + mv mountdir/hello.txt mountdir/hello2.txt + test_must_fail ipfs cat "$HASH" >/dev/null + ' + + test_expect_success "okay again after moving back" ' + mv mountdir/hello2.txt mountdir/hello.txt && + ipfs cat "$HASH" >/dev/null + ' + + test_expect_success "fail after file move" ' + mv mountdir/hello.txt mountdir/hello2.txt + test_must_fail ipfs cat "$HASH" >/dev/null + ' + + test_expect_success "okay after re-adding under new name" ' + ipfs $cmd "$dir"/mountdir/hello2.txt 2> add.output && + ipfs cat "$HASH" >/dev/null + ' + + test_expect_success "restore state" ' + mv mountdir/hello2.txt mountdir/hello.txt && + ipfs $cmd "$dir"/mountdir/hello.txt 2> add.output && + ipfs cat "$HASH" >/dev/null + ' + + test_expect_success "fail after file change" ' + # note: filesize shrinks + echo "hello world!" >mountdir/hello.txt && + test_must_fail ipfs cat "$HASH" >cat.output + ' + + test_expect_success "fail after file change, same size" ' + # note: filesize does not change + echo "HELLO WORLDS!" >mountdir/hello.txt && + test_must_fail ipfs cat "$HASH" >cat.output + ' +} + +test_add_cat_5MB() { + cmd=$1 + dir=$2 + HASH=$3 # "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + + test_expect_success "generate 5MB file using go-random" ' + random 5242880 41 >mountdir/bigfile + ' + + test_expect_success "sha1 of the file looks ok" ' + echo "11145620fb92eb5a49c9986b5c6844efda37e471660e" >sha1_expected && + multihash -a=sha1 -e=hex mountdir/bigfile >sha1_actual && + test_cmp sha1_expected sha1_actual + ' + + test_expect_success "'ipfs $cmd bigfile' succeeds" ' + ipfs $cmd "$dir"/mountdir/bigfile >actual + ' + + test_expect_success "'ipfs $cmd bigfile' output looks good" ' + echo "added $HASH "$dir"/mountdir/bigfile" >expected && + test_cmp expected actual + ' + + test_expect_success "'ipfs cat' succeeds" ' + ipfs cat "$HASH" >actual + ' + + test_expect_success "'ipfs cat' output looks good" ' + test_cmp mountdir/bigfile actual + ' +} + +test_add_cat_200MB() { + cmd=$1 + dir=$2 + HASH=$3 #"QmVbVLFLbz72tRSw3HMBh6ABKbRVavMQLoh2BzQ4dUSAYL" + + test_expect_success "generate 200MB file using go-random" ' + random 209715200 41 >mountdir/hugefile + ' + + test_expect_success "sha1 of the file looks ok" ' + echo "11146a3985bff32699f1874517ad0585bbd280efc1de" >sha1_expected && + multihash -a=sha1 -e=hex mountdir/hugefile >sha1_actual && + test_cmp sha1_expected sha1_actual + ' + + test_expect_success "'ipfs $cmd hugefile' succeeds" ' + ipfs $cmd "$dir"/mountdir/hugefile >actual + ' + + test_expect_success "'ipfs $cmd hugefile' output looks good" ' + echo "added $HASH "$dir"/mountdir/hugefile" >expected && + test_cmp expected actual + ' + + test_expect_success "'ipfs cat' succeeds" ' + ipfs cat "$HASH" >actual + ' + + test_expect_success "'ipfs cat' output looks good" ' + test_cmp mountdir/hugefile actual + ' + + test_expect_success "fail after file rm" ' + rm mountdir/hugefile actual && + test_must_fail ipfs cat "$HASH" >/dev/null + ' +} + +test_add_mulpl_files() { + cmd=$1 + + test_expect_success "generate directory with several files" ' + mkdir adir && + echo "file1" > adir/file1 && + echo "file2" > adir/file2 && + echo "file3" > adir/file3 + ' + + dir="`pwd`"/adir + test_expect_success "add files by listing them all on command line" ' + ipfs $cmd "$dir"/file1 "$dir"/file2 "$dir"/file3 > add-expect + ' + + test_expect_success "all files added" ' + grep file1 add-expect && + grep file2 add-expect && + grep file3 add-expect + ' + + test_expect_success "cleanup" ' + rm -r adir + ' +} + +filestore_test_exact_paths() { + opt=$1 + + test_expect_success "prep for path checks" ' + mkdir mydir && + ln -s mydir dirlink && + echo "Hello Worlds!!" > dirlink/hello.txt + ' + + test_expect_success "ipfs filestore add $opts adds under the expected path name (with symbolic links)" ' + FILEPATH="`pwd`/dirlink/hello.txt" && + HASH=`ipfs filestore add $opt "$FILEPATH" -q` && + echo "$FILEPATH" > ls-expected && + ipfs filestore ls-files -q $HASH > ls-actual && + test_cmp ls-expected ls-actual + ' + + test_expect_success "ipfs filestore ls dirlink/ works as expected" ' + echo "$HASH" > ls-expected + ipfs filestore ls -q "`pwd`/dirlink/" > ls-actual + test_cmp ls-expected ls-actual + ' + + test_expect_success "ipfs filestore add $opts --physical works as expected" ' + ipfs filestore rm $HASH && + ( cd dirlink && + ipfs filestore add $opt --physical hello.txt + FILEPATH="`pwd -P`/hello.txt" && + echo "$FILEPATH" > ls-expected && + ipfs filestore ls-files -q $HASH > ls-actual && + test_cmp ls-expected ls-actual ) + ' + + test_expect_success "ipfs filestore add $opts --logical works as expected" ' + ipfs filestore rm $HASH && + ( cd dirlink && + ipfs filestore add $opt --logical hello.txt + FILEPATH="`pwd -L`/hello.txt" && + echo "$FILEPATH" > ls-expected && + ipfs filestore ls-files -q $HASH > ls-actual && + test_cmp ls-expected ls-actual ) + ' + + test_expect_success "cleanup from path checks" ' + ipfs filestore rm $HASH && + rm -rf mydir + ' +} + +test_add_symlinks() { + opt=$1 + + test_expect_success "creating files with symbolic links succeeds" ' + rm -rf files && + mkdir -p files/foo && + mkdir -p files/bar && + echo "some text" > files/foo/baz && + ln -s files/foo/baz files/bar/baz && + ln -s files/does/not/exist files/bad + ' + + test_expect_success "adding a symlink adds the link itself" ' + ipfs filestore add --logical -q $opt files/bar/baz > goodlink_out + ' + + test_expect_success "output looks good" ' + echo "QmdocmZeF7qwPT9Z8SiVhMSyKA2KKoA2J7jToW6z6WBmxR" > goodlink_exp && + test_cmp goodlink_exp goodlink_out + ' + + test_expect_success "adding a broken symlink works" ' + ipfs filestore add --logical -q $opt files/bad > badlink_out + ' + + test_expect_success "output looks good" ' + echo "QmWYN8SEXCgNT2PSjB6BnxAx6NJQtazWoBkTRH9GRfPFFQ" > badlink_exp && + test_cmp badlink_exp badlink_out + ' +} + +test_add_symlinks_fails_cleanly() { + opt=$1 + + test_expect_success "creating files with symbolic links succeeds" ' + rm -rf files && + mkdir -p files/foo && + mkdir -p files/bar && + echo "some text" > files/foo/baz && + ln -s files/foo/baz files/bar/baz && + ln -s files/does/not/exist files/bad + ' + + test_expect_success "adding a symlink fails cleanly" ' + test_must_fail ipfs filestore add --logical -q $opt files/bar/baz > goodlink_out + ' + + test_expect_success "ipfs daemon did not crash" ' + kill -0 $IPFS_PID + ' + + test_expect_success "adding a broken link fails cleanly" ' + test_must_fail ipfs filestore add --logical -q $opt files/bad > badlink_out + ' + + test_expect_success "ipfs daemon did not crash" ' + kill -0 $IPFS_PID + ' +} + +test_add_dir_w_symlinks() { + opt=$1 + + test_expect_success "adding directory with symlinks in it works" ' + ipfs filestore add --logical -q -r $opt files/ > dirlink_out + ' +} + +# must do with the daemon offline +reset_filestore() { + test_expect_success "resting filestore" ' + rm -r .ipfs/filestore-db && + ipfs filestore enable + ' +} + +filestore_test_w_daemon() { + opt=$1 + + test_init_ipfs + + test_launch_ipfs_daemon $opt + + test_expect_success "can't enable filestore while daemon is running" ' + test_must_fail ipfs filestore enable + ' + + test_kill_ipfs_daemon + + test_enable_filestore + + test_launch_ipfs_daemon $opt + + test_add_cat_file "filestore add " "`pwd`" "QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + + test_post_add "filestore add " "`pwd`" + + test_add_empty_file "filestore add " "`pwd`" + + test_add_cat_5MB "filestore add " "`pwd`" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + + test_add_mulpl_files "filestore add " + + test_expect_success "testing filestore add -r should fail" ' + mkdir adir && + echo "Hello Worlds!" > adir/file1 && + echo "HELLO WORLDS!" > adir/file2 && + random 5242880 41 > adir/file3 && + test_must_fail ipfs filestore add -r "`pwd`/adir" + ' + rm -rf adir + + test_add_symlinks_fails_cleanly + + filestore_test_exact_paths + + test_expect_success "ipfs add -S fails unless enable" ' + echo "Hello Worlds!" >mountdir/hello.txt && + test_must_fail ipfs filestore add -S "`pwd`"/mountdir/hello.txt >actual + ' + + test_expect_success "filestore mv should fail" ' + HASH=QmQHRQ7EU8mUXLXkvqKWPubZqtxYPbwaqYo6NXSfS9zdCc && + random 5242880 42 >mountdir/bigfile-42 && + ipfs add mountdir/bigfile-42 && + test_must_fail ipfs filestore mv $HASH "`pwd`/mountdir/bigfile-42-also" + ' + + test_kill_ipfs_daemon + + reset_filestore + + #test_expect_success "clean filestore" ' + # ipfs filestore ls -q | xargs ipfs filestore rm && + # test -z "`ipfs filestore ls -q`" + #' + + test_expect_success "enable Filestore.APIServerSidePaths" ' + ipfs config Filestore.APIServerSidePaths --bool true + ' + + test_launch_ipfs_daemon $opt + + test_add_cat_file "filestore add -S" "`pwd`" "QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + + test_post_add "filestore add -S" "`pwd`" + + test_add_empty_file "filestore add -S" "`pwd`" + + test_add_cat_5MB "filestore add -S" "`pwd`" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + + test_add_mulpl_files "filestore add -S" + + cat < add_expect +added QmQhAyoEzSg5JeAzGDCx63aPekjSGKeQaYs4iRf4y6Qm6w adir +added QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb `pwd`/adir/file3 +added QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH `pwd`/adir/file1 +added QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN `pwd`/adir/file2 +EOF + + test_expect_success "testing filestore add -S -r" ' + mkdir adir && + echo "Hello Worlds!" > adir/file1 && + echo "HELLO WORLDS!" > adir/file2 && + random 5242880 41 > adir/file3 && + ipfs filestore add -S -r "`pwd`/adir" | LC_ALL=C sort > add_actual && + test_cmp add_expect add_actual && + ipfs cat QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH > cat_actual + test_cmp adir/file1 cat_actual + ' + + test_expect_failure "filestore mv" ' + HASH=QmQHRQ7EU8mUXLXkvqKWPubZqtxYPbwaqYo6NXSfS9zdCc && + test_must_fail ipfs filestore mv $HASH "mountdir/bigfile-42-also" && + ipfs filestore mv $HASH "`pwd`/mountdir/bigfile-42-also" + ' + + filestore_test_exact_paths '-S' + + test_add_symlinks '-S' + + test_add_dir_w_symlinks '-S' + + test_kill_ipfs_daemon + +} + diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 49d2db7178f..49d7f40e236 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -302,7 +302,7 @@ test_expect_success "'ipfs add' with stdin input succeeds" ' test_expect_success "'ipfs add' output looks good" ' HASH="QmZDhWpi8NvKrekaYYhxKCdNVGWsFFe1CREnAjP1QbPaB3" && - echo "added $HASH $HASH" >expected && + echo "added $HASH " >expected && test_cmp expected actual ' diff --git a/test/sharness/t0046-multifile-add.sh b/test/sharness/t0046-multifile-add.sh new file mode 100755 index 00000000000..6dafd0a9076 --- /dev/null +++ b/test/sharness/t0046-multifile-add.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test add and cat commands" + +. lib/test-lib.sh + +test_init_ipfs + +test_expect_success "create some files" ' + echo A > fileA && + echo B > fileB && + echo C > fileC +' + +test_expect_success "add files all at once" ' + ipfs add -q fileA fileB fileC > hashes +' + +test_expect_success "unpin one of the files" ' + ipfs pin rm `head -1 hashes` > pin-out +' + +test_expect_success "unpin output looks good" ' + echo "unpinned `head -1 hashes`" > pin-expect + test_cmp pin-expect pin-out +' + +test_expect_success "create files with same name but in different directories" ' + mkdir dirA && + mkdir dirB && + echo AA > dirA/fileA && + echo BA > dirB/fileA +' + +test_expect_success "add files with same name but in different directories" ' + ipfs add -q dirA/fileA dirB/fileA > hashes +' + +cat < cat-expected +AA +BA +EOF + +test_expect_success "check that both files are added" ' + cat hashes | xargs ipfs cat | LC_ALL=C sort > cat-actual + test_cmp cat-expected cat-actual +' + +test_expect_success "adding multiple directories fails cleanly" ' + test_must_fail ipfs add -q -r dirA dirB +' + +test_expect_success "adding multiple directories with -w is okay" ' + ipfs add -q -r -w dirA dirB > hashes && + ipfs ls `tail -1 hashes` > ls-res +' + +test_done diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index e4895173625..2af3506fb39 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -29,11 +29,6 @@ test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' -test_expect_success "'ipfs repo gc' looks good (patch root)" ' - PATCH_ROOT=QmQXirSbubiySKnqaFyfs5YzziXRB5JEVQVjU6xsd7innr && - grep "removed $PATCH_ROOT" gc_out_actual -' - test_expect_success "'ipfs repo gc' doesnt remove file" ' ipfs cat "$HASH" >out && test_cmp out afile @@ -104,8 +99,7 @@ test_expect_success "remove direct pin" ' test_expect_success "'ipfs repo gc' removes file" ' ipfs repo gc >actual7 && - grep "removed $HASH" actual7 && - grep "removed $PATCH_ROOT" actual7 + grep "removed $HASH" actual7 ' test_expect_success "'ipfs refs local' no longer shows file" ' @@ -114,8 +108,7 @@ test_expect_success "'ipfs refs local' no longer shows file" ' grep "QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y" actual8 && grep "$EMPTY_DIR" actual8 && grep "$HASH_WELCOME_DOCS" actual8 && - test_must_fail grep "$HASH" actual8 && - test_must_fail grep "$PATCH_ROOT" actual8 + test_must_fail grep "$HASH" actual8 ' test_expect_success "adding multiblock random file succeeds" ' diff --git a/test/sharness/t0235-cli-request.sh b/test/sharness/t0235-cli-request.sh index 9c28843e824..9a9370e3f82 100755 --- a/test/sharness/t0235-cli-request.sh +++ b/test/sharness/t0235-cli-request.sh @@ -19,7 +19,7 @@ test_expect_success "output does not contain multipart info" ' test_expect_code 1 grep multipart nc_out ' -test_expect_success "request looks good" ' +test_expect_failure "request looks good" ' grep "POST /api/v0/cat" nc_out ' diff --git a/test/sharness/t0260-filestore.sh b/test/sharness/t0260-filestore.sh new file mode 100755 index 00000000000..9ef8fbc54cb --- /dev/null +++ b/test/sharness/t0260-filestore.sh @@ -0,0 +1,292 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_expect_success "can't use filestore unless it is enabled" ' + test_must_fail ipfs filestore ls +' + +test_enable_filestore + +test_add_cat_file "filestore add" "`pwd`" "QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + +test_post_add "filestore add" "`pwd`" + +test_add_cat_file "filestore add --raw-leaves" "`pwd`" "zdvgqC4vX1j7higiYBR1HApkcjVMAFHwJyPL8jnKK6sVMqd1v" + +test_post_add "filestore add --raw-leaves" "`pwd`" + +test_add_empty_file "filestore add" "`pwd`" + +test_add_cat_5MB "filestore add" "`pwd`" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + +test_add_cat_5MB "filestore add --raw-leaves" "`pwd`" "QmefsDaD3YVphd86mxjJfPLceKv8by98aB6J6sJxK13xS2" + +test_add_mulpl_files "filestore add" + +test_expect_success "fail after file move" ' + mv mountdir/bigfile mountdir/bigfile2 + test_must_fail ipfs cat "$HASH" >/dev/null +' + +# check "ipfs filestore " cmd by using state left by add commands + +cat < ls_expect_all +QmQ8jJxa1Ts9fKsyUXcdYRHHUkuhJ69f82CF8BNX14ovLT +QmQNcknfZjsABxg2bwxZQ9yqoUZW5dtAfCK3XY4eadjnxZ +QmQnNhFzUjVRMHxafWaV2z7XZV8no9xJTdybMZbhgZ7776 +QmSY1PfYxzxJfQA3A19NdZGAu1fZz33bPGAhcKx82LMRm2 +QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb +QmTFH6xrLxiwC7WRwou2QkvgZwVSdQNHc1uGfPDNBqH2rK +QmTbkLhagokC5NsxRLC2fanadqzFdTCdBB7cJWCg3U2tgL +QmTvvmPaPBHRAo2CTvQC6VRYJaMwsFigDbsqhRjLBDypAa +QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH +QmWgZKyDJzixHydY5toiJ2EHFdDkooWJnvH5uixY4rhq2W +QmYNVKQFvW3UwUDGoGSS68eBBYSuFY8RVp7UTinkY8zkYv +QmZBe6brSjd2XBzAyqJRAYnNm3qRYR4BXk8Akfuot7fuSY +QmayX17gA63WRmcZkQuJGcDAv1hWP4ULpXaPUHSf7J6UbC +Qmb6wyFUBKshoaFRfh3xsdbrRF9WA5sdp62R6nWEtgjSEK +QmcZm5DH1JpbWkNnXsCXMioaQzXqcq7AmoQ3BK5Q9iWXJc +Qmcp8vWcq2xLnAum4DPqf3Pfr2Co9Hsj7kxkg4FxUAC4EE +QmeXTdS4ZZ99AcTg6w3JwndF3T6okQD17wY1hfRR7qQk8f +QmeanV48k8LQxWMY1KmoSAJiF6cSm1DtCsCzB5XMbuYNeZ +Qmej7SUFGehBVajSUpW4psbrMzcSC9Zip9awX9anLvofyZ +QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp +QmfAGX7cH2G16Wb6tzVgVjwJtphCz3SeuRqvFmGuVY3C7D +QmfYBbC153rBir5ECS2rzrKVUEer6rgqbRpriX2BviJHq1 +EOF + +cat < ls_expect +QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb +QmUtkGLvPf63NwVzLPKPUYgwhn8ZYPWF6vKWN3fZ2amfJF +QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH +QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH +Qmae3RedM7SNkWGsdzYzsr6svmsFdsva4WoTvYYsWhUSVz +QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH +QmefsDaD3YVphd86mxjJfPLceKv8by98aB6J6sJxK13xS2 +Qmesmmf1EEG1orJb6XdK6DabxexsseJnCfw8pqWgonbkoj +zdvgqC4vX1j7higiYBR1HApkcjVMAFHwJyPL8jnKK6sVMqd1v +zdvgqC4vX1j7higiYBR1HApkcjVMAFHwJyPL8jnKK6sVMqd1v +EOF + +test_expect_success "testing filestore ls" ' + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual_all && + ipfs filestore ls -q | LC_ALL=C sort > ls_actual && + test_cmp ls_expect ls_actual +' +test_expect_success "testing filestore verify" ' + test_must_fail ipfs filestore verify > verify_actual && + grep -q "changed QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" verify_actual && + grep -q "no-file QmQ8jJxa1Ts9fKsyUXcdYRHHUkuhJ69f82CF8BNX14ovLT" verify_actual && + grep -q "problem QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" verify_actual && + grep -q "ok $EMPTY_HASH" verify_actual +' + +test_expect_success "tesing re-adding file after change" ' + ipfs filestore add "`pwd`"/mountdir/hello.txt && + ipfs filestore ls -q -a | grep -q QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +cat < ls_expect +QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH +QmefsDaD3YVphd86mxjJfPLceKv8by98aB6J6sJxK13xS2 +EOF + +test_expect_success "testing filestore clean invalid" ' + ipfs filestore clean invalid > rm-invalid-output && + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual && + test_cmp ls_expect ls_actual +' + +cat < ls_expect +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH +EOF + +test_expect_success "testing filestore clean incomplete" ' + ipfs filestore clean incomplete > rm-invalid-output && + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual && + test_cmp ls_expect ls_actual +' + +test_expect_success "re-added file still available" ' + ipfs cat QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > expected && + test_cmp expected mountdir/hello.txt +' + +test_expect_success "testing filestore rm" ' + ipfs filestore rm QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +test_expect_success "testing file removed" ' + test_must_fail ipfs cat QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > expected +' + +test_add_cat_5MB "filestore add" "`pwd`" "QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb" + +test_expect_success "testing filestore rm -r" ' + ipfs filestore rm -r QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb > rm_actual +' + +cat < ls_expect +QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH +EOF + +test_expect_success "testing file removed" ' + ipfs filestore ls -q -a | LC_ALL=C sort > ls_actual && + test_cmp ls_expect ls_actual +' + +# +# filestore_test_exact_paths +# + +filestore_test_exact_paths + +# +# Duplicate block and pin testing +# + +test_expect_success "make sure block doesn't exist" ' + test_must_fail ipfs cat QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN && + ipfs filestore ls QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > res && + test ! -s res +' + +test_expect_success "create filestore block" ' + ipfs filestore add --logical mountdir/hello.txt && + ipfs cat QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +test_expect_success "add duplicate block with --allow-dup" ' + ipfs add --allow-dup mountdir/hello.txt +' + +test_expect_success "add unpinned duplicate block" ' + echo "Hello Mars!" > mountdir/hello2.txt && + ipfs add --pin=false mountdir/hello2.txt && + ipfs filestore add --logical mountdir/hello2.txt +' + +cat < locate_expect0 +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN /blocks found +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN /filestore found +EOF + +test_expect_success "ipfs block locate" ' + ipfs block locate QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > locate_actual0 && + test_cmp locate_expect0 locate_actual0 +' + +test_expect_success "testing filestore dups pinned" ' + ipfs filestore dups pinned > dups-actual && + echo QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > dups-expected && + test_cmp dups-actual dups-expected +' + +test_expect_success "testing filestore dups unpinned" ' + ipfs filestore dups unpinned > dups-actual && + echo QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN > dups-expected && + test_cmp dups-actual dups-expected +' + +test_expect_success "testing filestore dups" ' + ipfs filestore dups > dups-out && + grep QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN dups-out && + grep QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN dups-out +' + +test_expect_success "ipfs block rm pinned but duplicate block" ' + ipfs block rm QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +cat < locate_expect1 +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN /blocks error blockstore: block not found +QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN /filestore found +EOF + +test_expect_success "ipfs block locate" ' + ipfs block locate QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN > locate_actual1 + test_cmp locate_expect1 locate_actual1 +' + +test_expect_success "ipfs filestore rm pinned block fails" ' + test_must_fail ipfs filestore rm QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +# +# Pin related tests +# + +clear_pins() { + test_expect_success "clearing all pins" ' + ipfs pin ls -q -t recursive > pin_ls && + ipfs pin ls -q -t direct >> pin_ls && + cat pin_ls | xargs ipfs pin rm > pin_rm && + ipfs pin ls -q > pin_ls && + test -e pin_ls -a ! -s pin_ls + ' +} + +cat < add_expect +added QmQhAyoEzSg5JeAzGDCx63aPekjSGKeQaYs4iRf4y6Qm6w adir +added QmSr7FqYkxYWGoSfy8ZiaMWQ5vosb18DQGCzjwEQnVHkTb `pwd`/adir/file3 +added QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH `pwd`/adir/file1 +added QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN `pwd`/adir/file2 +EOF + +reset_filestore +clear_pins + +test_expect_success "testing filestore add -r --pin" ' + mkdir adir && + echo "Hello Worlds!" > adir/file1 && + echo "HELLO WORLDS!" > adir/file2 && + random 5242880 41 > adir/file3 && + ipfs filestore add -r --pin "`pwd`"/adir | LC_ALL=C sort > add_actual && + test_cmp add_expect add_actual +' + +test_expect_success "testing rm of indirect pinned file" ' + test_must_fail ipfs filestore rm QmZm53sWMaAQ59x56tFox8X9exJFELWC33NLjK6m8H7CpN +' + +clear_pins + +test_expect_failure "testing filestore mv" ' + HASH=QmQHRQ7EU8mUXLXkvqKWPubZqtxYPbwaqYo6NXSfS9zdCc && + random 5242880 42 >mountdir/bigfile-42 && + ipfs add mountdir/bigfile-42 && + ipfs filestore mv $HASH mountdir/bigfile-42-also && + test_cmp mountdir/bigfile-42 mountdir/bigfile-42-also +' + +test_expect_failure "testing filestore mv result" ' + ipfs filestore verify -l9 QmQHRQ7EU8mUXLXkvqKWPubZqtxYPbwaqYo6NXSfS9zdCc > verify.out && + grep -q "ok \+QmQHRQ7EU8mUXLXkvqKWPubZqtxYPbwaqYo6NXSfS9zdCc " verify.out +' + +# +# Additional add tests +# + +test_add_symlinks + +test_add_dir_w_symlinks + +test_add_cat_200MB "filestore add" "`pwd`" "QmVbVLFLbz72tRSw3HMBh6ABKbRVavMQLoh2BzQ4dUSAYL" + +test_add_cat_200MB "filestore add --raw-leaves" "`pwd`" "QmYJWknpk2HUjVCkTDFMcTtxEJB4XbUpFRYW4BCAEfDN6t" + +test_done diff --git a/test/sharness/t0261-filestore-online.sh b/test/sharness/t0261-filestore-online.sh new file mode 100755 index 00000000000..135102a499d --- /dev/null +++ b/test/sharness/t0261-filestore-online.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +filestore_test_w_daemon + +test_done diff --git a/test/sharness/t0262-filestore-config.sh b/test/sharness/t0262-filestore-config.sh new file mode 100755 index 00000000000..e9df2b03ddc --- /dev/null +++ b/test/sharness/t0262-filestore-config.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_enable_filestore + +test_add_cat_file "filestore add" "`pwd`" "QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + +export IPFS_LOGGING=debug +export IPFS_LOGGING_FMT=nocolor + +HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" + +test_expect_success "file always checked" ' + ipfs config Filestore.Verify always 2> log && + ipfs cat "$HASH" 2> log && + grep -q "verifying block $HASH" log && + ! grep -q "updating block $HASH" log +' + +test_expect_success "file checked after change" ' + ipfs config Filestore.Verify ifchanged 2> log && + sleep 2 && # to accommodate systems without sub-second mod-times + echo "HELLO WORLDS!" >mountdir/hello.txt && + test_must_fail ipfs cat "$HASH" 2> log && + grep -q "verifying block $HASH" log && + grep -q "updating block $HASH" log +' + +test_expect_success "file never checked" ' + echo "Hello Worlds!" >mountdir/hello.txt && + ipfs add "$dir"/mountdir/hello.txt >actual 2> log && + ipfs config Filestore.Verify never 2> log && + echo "HELLO Worlds!" >mountdir/hello.txt && + ( ipfs cat "$HASH" || true ) 2> log && + grep -q "BlockService GetBlock" log && # Make sure we are still logging + ! grep -q "verifying block $HASH" log && + ! grep -q "updating block $HASH" log +' + +test_done diff --git a/test/sharness/t0263-filestore-gc.sh b/test/sharness/t0263-filestore-gc.sh new file mode 100755 index 00000000000..56e22d204a6 --- /dev/null +++ b/test/sharness/t0263-filestore-gc.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_enable_filestore + +# add block +# add filestore block / rm file +# make sure gc still words + +FILE1=QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG +test_expect_success "add a pinned file" ' + echo "Hello World!" > file1 && + ipfs add file1 + ipfs cat $FILE1 | cmp file1 +' + +FILE2=QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN +test_expect_success "add an unpinned file" ' + echo "Hello Mars!" > file2 + ipfs add --pin=false file2 + ipfs cat $FILE2 | cmp file2 +' + +FILE3=QmeV1kwh3333bsnT6YRfdCRrSgUPngKmAhhTa4RrqYPbKT +test_expect_success "add and pin a directory using the filestore" ' + mkdir adir && + echo "hello world!" > adir/file3 && + echo "hello mars!" > adir/file4 && + ipfs filestore add --logical -r --pin adir && + ipfs cat $FILE3 | cmp adir/file3 +' + +FILE5=QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4 +test_expect_success "add a unpinned file to the filestore" ' + echo "Hello Venus!" > file5 && + ipfs filestore add --logical --pin=false file5 && + ipfs cat $FILE5 | cmp file5 +' + +test_expect_success "make sure filestore block is really not pinned" ' + test_must_fail ipfs pin ls $FILE5 +' + +FILE6=QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH +test_expect_success "add a unpinned empty file to the filestore" ' + cat /dev/null > file6 && + ipfs filestore add --logical --pin=false file6 && + ipfs cat $FILE6 | cmp file6 +' + +test_expect_success "make sure empty filestore block is really not pinned" ' + test_must_fail ipfs pin ls $FILE6 +' + +test_expect_success "remove one of the backing files" ' + rm adir/file3 && + test_must_fail ipfs cat $FILE3 +' + +test_expect_success "make ipfs pin ls is still okay" ' + ipfs pin ls +' + +test_expect_success "make sure the gc will still run" ' + ipfs repo gc +' + +test_expect_success "make sure pinned block still available after gc" ' + ipfs cat $FILE1 +' + +test_expect_success "make sure un-pinned block got removed" ' + test_must_fail ipfs cat $FILE2 +' + +test_expect_success "make sure unpinned filestore block did not get removed" ' + ipfs cat $FILE5 +' + +test_expect_success "check that we can remove an un-pinned filestore block" ' + ipfs filestore rm $FILE5 +' + +test_expect_success "make sure unpinned empty filestore block did not get removed" ' + ipfs cat $FILE6 +' + +test_expect_success "check that we can remove an empty un-pinned filestore block" ' + ipfs filestore rm $FILE6 +' + +test_done diff --git a/test/sharness/t0264-filestore-offline.sh b/test/sharness/t0264-filestore-offline.sh new file mode 100755 index 00000000000..76fa773bf89 --- /dev/null +++ b/test/sharness/t0264-filestore-offline.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +filestore_test_w_daemon --offline + +test_done diff --git a/test/sharness/t0265-filestore-verify.sh b/test/sharness/t0265-filestore-verify.sh new file mode 100755 index 00000000000..b1e4046dd1e --- /dev/null +++ b/test/sharness/t0265-filestore-verify.sh @@ -0,0 +1,466 @@ +#!/bin/sh +# +# Copyright (c) 2016 Kevin Atkinson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_enable_filestore + +test_verify_cmp() { + LC_ALL=C sort $1 | grep '[^[:blank:]]' > expect-sorted + LC_ALL=C sort $2 | grep '[^[:blank:]]' > actual-sorted + test_cmp expect-sorted actual-sorted +} + +######### +# +# Check append +# + +test_expect_success "create a file" ' + random 1000000 12 > afile && + HASH=$(ipfs filestore add -q --logical afile) +' + +test_expect_success "run ipfs verify" ' + ipfs filestore verify > verify-out && + fgrep -q "ok $HASH" verify-out +' + +test_expect_success "append to the file" ' + echo "more content" >> afile +' + +test_expect_success "test ipfs verify output" ' + ipfs filestore verify > verify-out && + fgrep -q "appended $HASH" verify-out +' + +test_expect_success "test ipfs verify -l0 output" ' + ipfs filestore verify -l0 > verify-out && + fgrep -q "complete $HASH" verify-out +' + +filestore_is_empty() { + ipfs filestore ls -q -a > should-be-empty && + test_cmp /dev/null should-be-empty +} + +######### +# +# Add a large enough file that the leaf node for the initial part of +# the file has a depth of at least two. Then, change the contents of +# the initial part and make sure "filestore clean full" is correct. +# + +reset_filestore + +test_expect_success "generate 200MB file using go-random" ' + random 209715200 41 >mountdir/hugefile +' + +test_expect_success "'ipfs add hugefile' succeeds" ' + HASH=$(ipfs filestore add -q --logical mountdir/hugefile) +' + +test_expect_success "change first bit of file" ' + dd conv=notrunc if=/dev/zero of=mountdir/hugefile count=1 +' + +cat < verify-expect +changed QmeomcMd37LRxkYn69XKiTpGEiJWRgUNEaxADx6ssfUJhp +problem QmRApadtoQSm9Bt3c2vVre7TKoQhh2LDFbaUV3So9yay9a +problem QmVbVLFLbz72tRSw3HMBh6ABKbRVavMQLoh2BzQ4dUSAYL + +EOF + +test_expect_success "ipfs verify produces expected output" ' + ipfs filestore verify -q > verify-actual || true && + test_verify_cmp verify-expect verify-actual +' + +test_expect_success "ipfs verify --post-orphan produces expected output" ' + ipfs filestore verify --post-orphan -q > verify-actual || true && + test_verify_cmp verify-expect verify-actual +' + +test_expect_success "'filestore clean full' is complete" ' + ipfs filestore clean full > clean-res && + filestore_is_empty +' + +######### +# +# Create a filestore with various problems and then make sure +# "filestore clean" handles them correctly +# + +cmp_verify() { + ipfs filestore verify -q > verify-actual + test_verify_cmp verify-now verify-actual +} + +cat < verify-initial +changed QmWZsU9wAHbaJHgCqFsDPRKomEcKZHei4rGNDrbjzjbmiJ +problem QmSLmxiETLJXJQxHBHwYd3BckDEqoZ3aZEnVGkb9EmbGcJ + +no-file QmXsjgFej1F7p6oKh4LSCscG9sBE8oBvV8foeC5791Goof +no-file QmXdpFugYKSCcXrRpWpqNPX9htvfYD81w38VcHyeMCD2gt +no-file QmepFjJy8jMuFs8bGbjPSUwnBD2542Hchwh44dvcfBdNi1 +no-file QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +problem QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +missing QmQVwjbNQZRpNoeTYwDwtA3CEEXHBeuboPgShqspGn822N +incomplete QmWRhUdTruDSjcdeiFjkGFA6Qd2JXfmVLNMM4KKjJ84jSD + +no-file QmXmiSYiAMd5vv1BgLnCVrgmqserFDAqcKGCBXmdWFHtcR +no-file QmNN38uhkUknjUAfWjYHg4E2TPjYaHuecTQTvT1gnUT3Je +no-file QmV5MfoytVQi4uGeATfpJvvzofXUe9MQ2Ymm5y3F3ZpqUc +no-file QmWThuQjx96vq9RphjwAqDQFndjTo1hFYXZwEJbcUjbo37 +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh + +orphan QmWuBmMUbJBjfoG8BgPAdVLuvtk8ysZuMrAYEFk18M9gvR +orphan Qmctvse35eQ8cEgUEsBxJYi7e4uNz3gnUqYYj8JTvZNY2A +orphan QmeoJhPxZ5tQoCXR2aMno63L6kJDbCJ3fZH4gcqjD65aKR +orphan QmVBGAJY8ghCXomPmttGs7oUZkQQUAKG3Db5StwneJtxwq +changed QmPSxQ4mNyq2b1gGu7Crsf3sbdSnYnFB3spSVETSLhD5RW +orphan QmPSxQ4mNyq2b1gGu7Crsf3sbdSnYnFB3spSVETSLhD5RW +EOF + +overlap_prep() { + reset_filestore + + test_expect_success "generate a bunch of file with some block sharing" ' + random 1000000 1 > a && + random 1000000 2 > b && + random 1000000 3 > c && + random 1000000 4 > d && + random 1000000 5 > e && + random 1000000 6 > f && + cat a b > ab && + cat b c > bc && + cp f f2 && + random 262144 10 > a1 && # a single block + random 262144 11 > a2 && # a single block + cat a1 a2 > a3 # when added will use the same block from a1 and a2 + ' + + test_expect_success "add files with overlapping blocks" ' + A_HASH=$(ipfs filestore add --logical -q a) && + AB_HASH=$(ipfs filestore add --logical -q ab) && + BC_HASH=$(ipfs filestore add --logical -q bc) && + B_HASH=$(ipfs filestore add --logical -q b) && + C_HASH=$(ipfs filestore add --logical -q c) && # note blocks of C not shared due to alignment + D_HASH=$(ipfs filestore add --logical -q d) && + E_HASH=$(ipfs filestore add --logical -q e) && + F_HASH=$(ipfs filestore add --logical -q f) && + ipfs filestore add --logical -q f2 && + A1_HASH=$(ipfs filestore add --logical -q a1) && + A2_HASH=$(ipfs filestore add --logical -q a2) && + A3_HASH=$(ipfs filestore add --logical -q a3) + ' + # Note: $A1_HASH and $A2_HASH are both have two entries, one of them + # in the root for the file a1 and a2 respectively and the other is a + # leaf for the file a3. +} + +interesting_prep() { + overlap_prep + + test_expect_success "create various problems" ' + # removing the backing file for a + rm a && + # remove the root to b + ipfs filestore rm $B_HASH && + # remove a block in c + ipfs filestore rm --allow-non-roots QmQVwjbNQZRpNoeTYwDwtA3CEEXHBeuboPgShqspGn822N && + # modify d + dd conv=notrunc if=/dev/zero of=d count=1 && + # modify e amd remove the root from the filestore creating a block + # that is both an orphan and "changed" + dd conv=notrunc if=/dev/zero of=e count=1 && + ipfs filestore rm $E_HASH && + # remove the backing file for f + rm f + ' + + test_expect_success "'filestore verify' produces expected output" ' + cp verify-initial verify-now && + cmp_verify + ' +} + +interesting_prep + +cat < verify-now +changed QmWZsU9wAHbaJHgCqFsDPRKomEcKZHei4rGNDrbjzjbmiJ +problem QmSLmxiETLJXJQxHBHwYd3BckDEqoZ3aZEnVGkb9EmbGcJ + +no-file QmXsjgFej1F7p6oKh4LSCscG9sBE8oBvV8foeC5791Goof +no-file QmXdpFugYKSCcXrRpWpqNPX9htvfYD81w38VcHyeMCD2gt +no-file QmepFjJy8jMuFs8bGbjPSUwnBD2542Hchwh44dvcfBdNi1 +no-file QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +problem QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +missing QmQVwjbNQZRpNoeTYwDwtA3CEEXHBeuboPgShqspGn822N +incomplete QmWRhUdTruDSjcdeiFjkGFA6Qd2JXfmVLNMM4KKjJ84jSD + +no-file QmXmiSYiAMd5vv1BgLnCVrgmqserFDAqcKGCBXmdWFHtcR +no-file QmNN38uhkUknjUAfWjYHg4E2TPjYaHuecTQTvT1gnUT3Je +no-file QmV5MfoytVQi4uGeATfpJvvzofXUe9MQ2Ymm5y3F3ZpqUc +no-file QmWThuQjx96vq9RphjwAqDQFndjTo1hFYXZwEJbcUjbo37 +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh +EOF +test_expect_success "'filestore clean orphan' (should remove 'changed' orphan)" ' + ipfs filestore clean orphan && + cmp_verify +' + +cat < verify-now +changed QmWZsU9wAHbaJHgCqFsDPRKomEcKZHei4rGNDrbjzjbmiJ +problem QmSLmxiETLJXJQxHBHwYd3BckDEqoZ3aZEnVGkb9EmbGcJ + +no-file QmXsjgFej1F7p6oKh4LSCscG9sBE8oBvV8foeC5791Goof +no-file QmXdpFugYKSCcXrRpWpqNPX9htvfYD81w38VcHyeMCD2gt +no-file QmepFjJy8jMuFs8bGbjPSUwnBD2542Hchwh44dvcfBdNi1 +no-file QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +problem QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +no-file QmXmiSYiAMd5vv1BgLnCVrgmqserFDAqcKGCBXmdWFHtcR +no-file QmNN38uhkUknjUAfWjYHg4E2TPjYaHuecTQTvT1gnUT3Je +no-file QmV5MfoytVQi4uGeATfpJvvzofXUe9MQ2Ymm5y3F3ZpqUc +no-file QmWThuQjx96vq9RphjwAqDQFndjTo1hFYXZwEJbcUjbo37 +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh + +orphan QmYswupx1AdGdTn6GeXVdaUBEe6rApd7GWSnobcuVZjeRV +orphan QmfDSgGhGsEf7LHC6gc7FbBMhGuYzxTLnbKqFBkWhGt8Qp +orphan QmSWnPbrLFmxfJ9vj2FvKKpVmu3SZprbt7KEbkUVjy7bMD +EOF +test_expect_success "'filestore clean incomplete' (will create more orphans)" ' + ipfs filestore clean incomplete && + cmp_verify +' + +cat < verify-now +changed QmWZsU9wAHbaJHgCqFsDPRKomEcKZHei4rGNDrbjzjbmiJ +problem QmSLmxiETLJXJQxHBHwYd3BckDEqoZ3aZEnVGkb9EmbGcJ + +no-file QmXsjgFej1F7p6oKh4LSCscG9sBE8oBvV8foeC5791Goof +no-file QmXdpFugYKSCcXrRpWpqNPX9htvfYD81w38VcHyeMCD2gt +no-file QmepFjJy8jMuFs8bGbjPSUwnBD2542Hchwh44dvcfBdNi1 +no-file QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +problem QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +no-file QmXmiSYiAMd5vv1BgLnCVrgmqserFDAqcKGCBXmdWFHtcR +no-file QmNN38uhkUknjUAfWjYHg4E2TPjYaHuecTQTvT1gnUT3Je +no-file QmV5MfoytVQi4uGeATfpJvvzofXUe9MQ2Ymm5y3F3ZpqUc +no-file QmWThuQjx96vq9RphjwAqDQFndjTo1hFYXZwEJbcUjbo37 +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh +EOF +test_expect_success "'filestore clean orphan'" ' + ipfs filestore clean orphan && + cmp_verify +' + +cat < verify-now +no-file QmXsjgFej1F7p6oKh4LSCscG9sBE8oBvV8foeC5791Goof +no-file QmXdpFugYKSCcXrRpWpqNPX9htvfYD81w38VcHyeMCD2gt +no-file QmepFjJy8jMuFs8bGbjPSUwnBD2542Hchwh44dvcfBdNi1 +no-file QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +problem QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +no-file QmXmiSYiAMd5vv1BgLnCVrgmqserFDAqcKGCBXmdWFHtcR +no-file QmNN38uhkUknjUAfWjYHg4E2TPjYaHuecTQTvT1gnUT3Je +no-file QmV5MfoytVQi4uGeATfpJvvzofXUe9MQ2Ymm5y3F3ZpqUc +no-file QmWThuQjx96vq9RphjwAqDQFndjTo1hFYXZwEJbcUjbo37 +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh + +orphan QmbZr7Fs6AJf7HpnTxDiYJqLXWDqAy3fKFXYVDkgSsH7DH +orphan QmToAcacDnpqm17jV7rRHmXcS9686Mk59KCEYGAMkh9qCX +orphan QmYtLWUVmevucXFN9q59taRT95Gxj5eJuLUhXKtwNna25t +EOF +test_expect_success "'filestore clean changed incomplete' (will create more orphans)" ' + ipfs filestore clean changed incomplete && + cmp_verify +' + +cat < verify-now +missing QmXWr5Td85uXqKhyL17uAsZ7aJZSvtXs3aMGTZ4wHvwubP +incomplete QmW6QuzoYEpwASzZktbc5G5Fkq3XeBbUfRCrrUEByYm6Pi + +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh + +orphan QmToAcacDnpqm17jV7rRHmXcS9686Mk59KCEYGAMkh9qCX +orphan QmbZr7Fs6AJf7HpnTxDiYJqLXWDqAy3fKFXYVDkgSsH7DH +orphan QmYtLWUVmevucXFN9q59taRT95Gxj5eJuLUhXKtwNna25t +EOF +test_expect_success "'filestore clean no-file' (will create an incomplete)" ' + ipfs filestore clean no-file && + cmp_verify +' + +cat < verify-final +ok QmaVeSKhGmPYxRyqA236Y4N5e4Rn6LGZKdCgaYUarEo5Nu + +ok QmcAkMdfBPYVzDCM6Fkrz1h8WXcprH8BLF6DmjNUGhXAnm + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmZcUeeYQEjDzbuEBhce8e7gTibUwotg3EvmSJ35gBxnZQ + +ok QmcSqqZ9CPrtf19jWM39geC5S1nqUtwytFt9dQ478hTkzt + +ok QmcTnu1vxYsCdbVjwpMQBr1cK1grmHNxG2bM17E1d4swpf + +ok QmeVzg9KFp8FswVxUN68xq8pHVXPR7wBcXXzNPLyqwzcxh +EOF +test_expect_success "'filestore clean incomplete orphan' (cleanup)" ' + cp verify-final verify-now && + ipfs filestore clean incomplete orphan && + cmp_verify +' + +# +# Now reset and redo with a full clean and should get the same results +# + +interesting_prep + +test_expect_success "'filestore clean full'" ' + cp verify-final verify-now && + ipfs filestore clean full && + cmp_verify +' + +test_expect_success "remove f from filestore" ' + ipfs filestore ls-files $F_HASH -q | grep "/f$" > filename && + ipfs filestore rm $F_HASH/"$(cat filename)//0" +' + +test_expect_success "make sure clean does not remove shared and valid blocks" ' + ipfs cat $AB_HASH > /dev/null + ipfs cat $BC_HASH > /dev/null + ipfs cat $F_HASH > /dev/null +' + +# +# Now reset and test "filestore rm -r" with overlapping files +# + +overlap_prep + +test_expect_success "remove bc, make sure b is still ok" ' + ipfs filestore rm -r $BC_HASH && + ipfs cat $B_HASH > /dev/null +' + +test_expect_success "remove a, make sure ab is still ok" ' + ipfs filestore rm -r $A_HASH && + ipfs cat $AB_HASH > /dev/null +' + +test_expect_success "remove just f, make sure f2 is still ok" ' + ipfs filestore ls-files $F_HASH -q | grep "/f$" > filename && + ipfs filestore rm -r $F_HASH/"$(cat filename)" + ipfs cat $F_HASH > /dev/null +' + +test_expect_success "add back f" ' + ipfs filestore add --logical f +' + +test_expect_success "completly remove f hash" ' + ipfs filestore rm -r $F_HASH > rm_actual && + grep "removed $F_HASH//.\+/f//0" rm_actual && + grep "removed $F_HASH//.\+/f2//0" rm_actual +' + +test_expect_success "remove a1 and a2" ' + test_must_fail ipfs filestore rm $A1_HASH $A2_HASH > rm_actual && + grep "removed $A1_HASH//.\+/a1//0" rm_actual && + grep "removed $A2_HASH//.\+/a2//0" rm_actual +' + +test_expect_success "make sure a3 is still okay" ' + ipfs cat $A3_HASH > /dev/null +' +test_done diff --git a/test/sharness/t0266-filestore-concurrent.sh b/test/sharness/t0266-filestore-concurrent.sh new file mode 100755 index 00000000000..d355eded7b3 --- /dev/null +++ b/test/sharness/t0266-filestore-concurrent.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# Copyright (c) 2014 Christian Couder +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test filestore" + +. lib/test-filestore-lib.sh +. lib/test-lib.sh + +test_init_ipfs + +test_enable_filestore + +export IPFS_LOGGING_FMT=nocolor + +test_launch_ipfs_daemon --offline + +test_expect_success "enable filestore debug logging" ' + ipfs log level filestore debug +' + +test_expect_success "generate 500MB file using go-random" ' + random 524288000 41 >mountdir/hugefile +' + +test_expect_success "'filestore clean orphan race condition" '( + set -m + (ipfs filestore add -q --logical mountdir/hugefile > hugefile-hash && echo "add done") & + sleep 1 && + (ipfs filestore clean orphan && echo "clean done") & + wait +)' + +test_kill_ipfs_daemon + +cat < filtered_expect +acquired add-lock refcnt now 1 +Starting clean operation. +Removing invalid blocks after clean. Online Mode. +released add-lock refcnt now 0 +EOF + +test_expect_success "filestore clean orphan race condition: operations ran in correct order" ' + egrep -i "add-lock|clean" daemon_err | cut -d " " -f 6- > filtered_actual && + test_cmp filtered_expect filtered_actual +' + +test_expect_success "filestore clean orphan race condition: file still accessible" ' + ipfs cat `cat hugefile-hash` > /dev/null +' + +reset_filestore + +export IPFS_FILESTORE_CLEAN_RM_DELAY=5s + +test_launch_ipfs_daemon --offline + +#test_expect_success "enable filestore debug logging" ' +# ipfs log level filestore debug +#' + +test_expect_success "ipfs add succeeds" ' + echo "Hello Worlds!" >mountdir/hello.txt && + ipfs filestore add --logical mountdir/hello.txt >actual && + HASH="QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH" && + ipfs cat "$HASH" > /dev/null +' + +test_expect_success "fail after file move" ' + mv mountdir/hello.txt mountdir/hello2.txt + test_must_fail ipfs cat "$HASH" >/dev/null +' + +test_expect_success "filestore clean invalid race condation" '( + set -m + ipfs filestore clean invalid > clean-actual & + sleep 2 && + ipfs filestore add --logical mountdir/hello2.txt && + wait +)' + +# FIXME: Instead test that the operations ran in the correct order +#test_expect_success "filestore clean race condation: output looks good" ' +# grep "cannot remove $HASH" clean-actual +#' + +test_expect_success "filestore clean race condation: file still available" ' + ipfs cat "$HASH" > /dev/null +' + +test_kill_ipfs_daemon + +unset IPFS_FILESTORE_CLEAN_RM_DELAY +export IPFS_FILESTORE_CLEAN_RM_DELAY + +test_expect_success "fail after file move" ' + rm mountdir/hugefile + test_must_fail ipfs cat `echo hugefile-hash` >/dev/null +' + +export IPFS_LOGGING=debug + +# note: exclusive mode deletes do not check if a DataObj has changed +# from under us and are thus likley to be faster when cleaning out +# a large number of invalid blocks +test_expect_failure "ipfs clean local mode uses exclusive mode" ' + ipfs filestore clean invalid > clean-out 2> clean-err && + grep "Exclusive Mode." clean-err +' + +test_done