fix!: return simple stats or extended stats (#760)

Splits the unixfs/mfs stat command return types into two types - "regular" stats (these only return stats available on the root node of the DAG) and "extended" - these collect stats from the DAG, potentially going to the network to fetch missing blocks.

There are separate simple and extended types for files and dirs, but raw/leaf nodes only get one type since there are never any nodes linked to from them.

As a bonus we can now calculate the size of directories correctly, (as the combination of all child files/directories), assuming all blocks are present in the blockstore or fetchable from the network.

Fixes #580

BREAKING CHANGE: Fields that would involve DAG traversal have been removed from the output of `fs.stat` - pass the `extended` option to have them returned

---------

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>
Co-authored-by: Daniel N <2color@users.noreply.github.com>

Author: achingbrain

packages/interop/package.json

@@ -73,12 +73,14 @@
     "@helia/unixfs": "^4.0.3",
     "@ipld/car": "^5.3.3",
     "@ipld/dag-cbor": "^9.2.2",
+    "@ipld/dag-pb": "^4.1.3",
     "@libp2p/crypto": "^5.0.7",
     "@libp2p/interface": "^2.2.1",
     "@libp2p/kad-dht": "^14.1.3",
     "@libp2p/keychain": "^5.0.10",
     "@libp2p/peer-id": "^5.0.8",
     "@libp2p/websockets": "^9.0.13",
+    "@multiformats/multiaddr": "^12.4.0",
     "@multiformats/sha3": "^3.0.2",
     "aegir": "^45.1.1",
     "helia": "^5.3.0",

packages/interop/src/unixfs-files.spec.ts

@@ -1,15 +1,20 @@
 /* eslint-env mocha */
 
 import { unixfs } from '@helia/unixfs'
+import * as dagPb from '@ipld/dag-pb'
+import { multiaddr } from '@multiformats/multiaddr'
 import { expect } from 'aegir/chai'
 import { fixedSize } from 'ipfs-unixfs-importer/chunker'
 import { balanced } from 'ipfs-unixfs-importer/layout'
+import drain from 'it-drain'
+import last from 'it-last'
 import { CID } from 'multiformats/cid'
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { createHeliaNode } from './fixtures/create-helia.js'
 import { createKuboNode } from './fixtures/create-kubo.js'
 import type { AddOptions, UnixFS } from '@helia/unixfs'
 import type { HeliaLibp2p } from 'helia'
-import type { ByteStream } from 'ipfs-unixfs-importer'
+import type { ByteStream, ImportCandidateStream } from 'ipfs-unixfs-importer'
 import type { KuboNode } from 'ipfsd-ctl'
 import type { AddOptions as KuboAddOptions } from 'kubo-rpc-client'
 
@@ -24,12 +29,32 @@ describe('@helia/unixfs - files', () => {
     return cid
   }
 
+  async function importDirectoryToHelia (data: ImportCandidateStream, opts?: Partial<AddOptions>): Promise<CID> {
+    const result = await last(unixFs.addAll(data, opts))
+
+    if (result == null) {
+      throw new Error('Nothing imported')
+    }
+
+    return CID.parse(result.cid.toString())
+  }
+
   async function importToKubo (data: ByteStream, opts?: KuboAddOptions): Promise<CID> {
     const result = await kubo.api.add(data, opts)
 
     return CID.parse(result.cid.toString())
   }
 
+  async function importDirectoryToKubo (data: ImportCandidateStream, opts?: KuboAddOptions): Promise<CID> {
+    const result = await last(kubo.api.addAll(data, opts))
+
+    if (result == null) {
+      throw new Error('Nothing imported')
+    }
+
+    return CID.parse(result.cid.toString())
+  }
+
   async function expectSameCid (data: () => ByteStream, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = {}): Promise<void> {
     const heliaCid = await importToHelia(data(), {
       // these are the default kubo options
@@ -85,4 +110,103 @@ describe('@helia/unixfs - files', () => {
 
     await expectSameCid(candidate)
   })
+
+  it('should return the same directory stats', async () => {
+    const candidates = [{
+      path: '/foo1.txt',
+      content: uint8ArrayFromString('Hello World!')
+    }, {
+      path: '/foo2.txt',
+      content: uint8ArrayFromString('Hello World!')
+    }]
+
+    const heliaCid = await importDirectoryToHelia(candidates, {
+      wrapWithDirectory: true
+    })
+    const kuboCid = await importDirectoryToKubo(candidates, {
+      cidVersion: 1,
+      chunker: `size-${1024 * 1024}`,
+      rawLeaves: true,
+      wrapWithDirectory: true
+    })
+
+    expect(heliaCid.toString()).to.equal(kuboCid.toString())
+
+    const heliaStat = await unixFs.stat(heliaCid, {
+      extended: true
+    })
+    const kuboStat = await kubo.api.files.stat(`/ipfs/${kuboCid}`, {
+      withLocal: true
+    })
+
+    expect(heliaStat.dagSize.toString()).to.equal(kuboStat.cumulativeSize.toString())
+    expect(heliaStat.dagSize.toString()).to.equal(kuboStat.sizeLocal?.toString())
+
+    // +1 because kubo doesn't count the root directory block
+    expect(heliaStat.blocks.toString()).to.equal((kuboStat.blocks + 1).toString())
+  })
+
+  it('fetches missing blocks during stat', async () => {
+    const chunkSize = 1024 * 1024
+    const size = chunkSize * 10
+
+    const candidate = (): ByteStream => (async function * () {
+      for (let i = 0; i < size; i += chunkSize) {
+        yield new Uint8Array(new Array(chunkSize).fill(0).map((val, index) => {
+          return Math.floor(Math.random() * 256)
+        }))
+      }
+    }())
+
+    const largeFileCid = await importToKubo(candidate())
+    const info = await kubo.info()
+
+    await helia.libp2p.dial(info.multiaddrs.map(ma => multiaddr(ma)))
+
+    // pull all blocks from kubo
+    await drain(unixFs.cat(largeFileCid))
+
+    // check the root block
+    const block = await helia.blockstore.get(largeFileCid)
+    const node = dagPb.decode(block)
+
+    expect(node.Links).to.have.lengthOf(40)
+
+    const stats = await unixFs.stat(largeFileCid, {
+      extended: true
+    })
+
+    expect(stats.unixfs?.fileSize()).to.equal(10485760n)
+    expect(stats.blocks).to.equal(41n)
+    expect(stats.dagSize).to.equal(10488250n)
+    expect(stats.localSize).to.equal(10485760n)
+
+    // remove one of the blocks so we now have an incomplete DAG
+    await helia.blockstore.delete(node.Links[0].Hash)
+
+    // block count and local file/dag sizes should be smaller
+    const updatedStats = await unixFs.stat(largeFileCid, {
+      extended: true,
+      offline: true
+    })
+
+    expect(updatedStats.unixfs?.fileSize()).to.equal(10485760n)
+    expect(updatedStats.blocks).to.equal(40n)
+    expect(updatedStats.dagSize).to.equal(10226092n)
+    expect(updatedStats.localSize).to.equal(10223616n)
+
+    await new Promise<void>((resolve) => {
+      setTimeout(() => {
+        resolve()
+      }, 1_000)
+    })
+
+    // block count and local file/dag sizes should be smaller
+    const finalStats = await unixFs.stat(largeFileCid, {
+      extended: true
+    })
+
+    // should have fetched missing block from Kubo
+    expect(finalStats).to.deep.equal(stats, 'did not fetch missing block')
+  })
 })

packages/mfs/src/index.ts

@@ -34,7 +34,7 @@ import { Key } from 'interface-datastore'
 import { UnixFS as IPFSUnixFS, type Mtime } from 'ipfs-unixfs'
 import { CID } from 'multiformats/cid'
 import { basename } from './utils/basename.js'
-import type { AddOptions, CatOptions, ChmodOptions, CpOptions, LsOptions, MkdirOptions as UnixFsMkdirOptions, RmOptions as UnixFsRmOptions, StatOptions, TouchOptions, UnixFS, UnixFSStats } from '@helia/unixfs'
+import type { AddOptions, CatOptions, ChmodOptions, CpOptions, LsOptions, MkdirOptions as UnixFsMkdirOptions, RmOptions as UnixFsRmOptions, StatOptions, TouchOptions, UnixFS, FileStats, DirectoryStats, RawStats, ExtendedStatOptions, ExtendedFileStats, ExtendedDirectoryStats, ExtendedRawStats } from '@helia/unixfs'
 import type { AbortOptions } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
 import type { Datastore } from 'interface-datastore'
@@ -213,7 +213,8 @@ export interface MFS {
    * console.info(stats)
    * ```
    */
-  stat(path: string, options?: Partial<StatOptions>): Promise<UnixFSStats>
+  stat(path: string, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  stat(path: string, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
 
   /**
    * Update the mtime of a UnixFS DAG in your MFS.
@@ -438,7 +439,9 @@ class DefaultMFS implements MFS {
     this.root = await this.#persistPath(trail, options)
   }
 
-  async stat (path: string, options?: Partial<StatOptions>): Promise<UnixFSStats> {
+  async stat (path: string, options?: StatOptions): Promise<FileStats | DirectoryStats | RawStats>
+  async stat (path: string, options?: ExtendedStatOptions): Promise<ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats>
+  async stat (path: string, options?: StatOptions | ExtendedStatOptions): Promise<FileStats | DirectoryStats | RawStats | ExtendedFileStats | ExtendedDirectoryStats | ExtendedRawStats> {
     const root = await this.#getRootCID()
 
     const trail = await this.#walkPath(root, path, {
@@ -453,9 +456,7 @@ class DefaultMFS implements MFS {
       throw new DoesNotExistError()
     }
 
-    return this.unixfs.stat(finalEntry.cid, {
-      ...options
-    })
+    return this.unixfs.stat(finalEntry.cid, options)
   }
 
   async touch (path: string, options?: Partial<TouchOptions>): Promise<void> {