Skip to content

Commit 9f51b17

Browse files
achingbrainachingbrain
authored
feat: expose extended option for ls in mfs and unixfs (#836)
Allows returning lightweight directory contents that does not require loading the root block of each directory entry.
1 parent 5a911c6 commit 9f51b17

7 files changed

Lines changed: 157 additions & 16 deletions

File tree

‎packages/mfs/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,9 @@
5959
"@libp2p/logger": "^5.1.4",
6060
"interface-blockstore": "^5.3.1",
6161
"interface-datastore": "^8.3.1",
62-
"ipfs-unixfs": "^11.2.0",
63-
"ipfs-unixfs-exporter": "^13.6.1",
64-
"ipfs-unixfs-importer": "^15.3.1",
62+
"ipfs-unixfs": "^11.2.5",
63+
"ipfs-unixfs-exporter": "^13.7.2",
64+
"ipfs-unixfs-importer": "^15.4.0",
6565
"multiformats": "^13.3.1"
6666
},
6767
"devDependencies": {

‎packages/mfs/src/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import type { AbortOptions } from '@libp2p/interface'
3939
import type { Blockstore } from 'interface-blockstore'
4040
import type { Datastore } from 'interface-datastore'
4141
import type { Mtime } from 'ipfs-unixfs'
42-
import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
42+
import type { UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'
4343
import type { ByteStream } from 'ipfs-unixfs-importer'
4444

4545
const log = logger('helia:mfs')
@@ -178,6 +178,7 @@ export interface MFS {
178178
* ```
179179
*/
180180
ls(path?: string, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry>
181+
ls(path: string, options: Partial<LsOptions> & { extended: false }): AsyncIterable<UnixFSBasicEntry>
181182

182183
/**
183184
* Make a new directory in your MFS.

‎packages/mfs/test/ls.spec.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,4 +177,67 @@ describe('ls', () => {
177177
expect(files.length).to.equal(1)
178178
expect(files.filter(file => file.name === fileName)).to.be.ok()
179179
})
180+
181+
it('should list a basic entry', async () => {
182+
const filePath = '/foo.txt'
183+
184+
await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), filePath, {
185+
rawLeaves: false,
186+
force: true
187+
})
188+
189+
const files = await all(fs.ls(filePath))
190+
191+
expect(files).to.have.nested.property('[0].type')
192+
expect(files).to.have.nested.property('[0].content')
193+
194+
const basicFiles = await all(fs.ls(filePath, {
195+
extended: false
196+
}))
197+
198+
expect(basicFiles).to.not.have.nested.property('[0].type')
199+
expect(basicFiles).to.not.have.nested.property('[0].content')
200+
})
201+
202+
it('lists basic files in a directory', async () => {
203+
const dirName = 'bar'
204+
const dirPath = `/${dirName}`
205+
const fileName = 'foo.txt'
206+
const filePath = `${dirPath}/${fileName}`
207+
208+
await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), filePath, {
209+
rawLeaves: false,
210+
force: true
211+
})
212+
213+
const files = await all(fs.ls(dirPath))
214+
215+
expect(files).to.have.nested.property('[0].type')
216+
expect(files).to.have.nested.property('[0].content')
217+
218+
const basicFiles = await all(fs.ls(dirPath, {
219+
extended: false
220+
}))
221+
222+
expect(basicFiles).to.not.have.nested.property('[0].type')
223+
expect(basicFiles).to.not.have.nested.property('[0].content')
224+
})
225+
226+
it('lists basic contents of a sharded directory', async () => {
227+
const shardedDirPath = '/sharded-dir'
228+
const shardedDirCid = await createShardedDirectory(blockstore)
229+
await fs.cp(shardedDirCid, shardedDirPath)
230+
231+
const files = await all(fs.ls(shardedDirPath))
232+
233+
expect(files).to.have.nested.property('[0].type')
234+
expect(files).to.have.nested.property('[0].content')
235+
236+
const basicFiles = await all(fs.ls(shardedDirPath, {
237+
extended: false
238+
}))
239+
240+
expect(basicFiles).to.not.have.nested.property('[0].type')
241+
expect(basicFiles).to.not.have.nested.property('[0].content')
242+
})
180243
})

‎packages/unixfs/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,9 +82,9 @@
8282
"@multiformats/murmur3": "^2.1.8",
8383
"hamt-sharding": "^3.0.6",
8484
"interface-blockstore": "^5.3.1",
85-
"ipfs-unixfs": "^11.2.0",
86-
"ipfs-unixfs-exporter": "^13.6.1",
87-
"ipfs-unixfs-importer": "^15.3.1",
85+
"ipfs-unixfs": "^11.2.5",
86+
"ipfs-unixfs-exporter": "^13.7.2",
87+
"ipfs-unixfs-importer": "^15.4.0",
8888
"it-all": "^3.0.6",
8989
"it-first": "^3.0.6",
9090
"it-glob": "^3.0.1",

‎packages/unixfs/src/commands/ls.ts‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { NoContentError, NotADirectoryError } from '../errors.js'
44
import { resolve } from './utils/resolve.js'
55
import type { LsOptions } from '../index.js'
66
import type { GetStore } from '../unixfs.js'
7-
import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
7+
import type { UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'
88
import type { CID } from 'multiformats/cid'
99

1010
const mergeOptions = mergeOpts.bind({ ignoreUndefined: true })
@@ -13,13 +13,29 @@ const defaultOptions: LsOptions = {
1313

1414
}
1515

16-
export async function * ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions> = {}): AsyncIterable<UnixFSEntry> {
16+
export function ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions & { extended: false }>): AsyncIterable<UnixFSBasicEntry>
17+
export function ls (cid: CID, blockstore: GetStore, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry>
18+
export async function * ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions> = {}): AsyncIterable<any> {
1719
const opts: LsOptions = mergeOptions(defaultOptions, options)
1820
const resolved = await resolve(cid, opts.path, blockstore, opts)
19-
const result = await exporter(resolved.cid, blockstore)
21+
const result = await exporter(resolved.cid, blockstore, {
22+
...options,
23+
extended: true
24+
})
2025

2126
if (result.type === 'file' || result.type === 'raw') {
22-
yield result
27+
if (options.extended === false) {
28+
const basic: UnixFSBasicEntry = {
29+
name: result.name,
30+
path: result.path,
31+
cid: result.cid
32+
}
33+
34+
yield basic
35+
} else {
36+
yield result
37+
}
38+
2339
return
2440
}
2541

@@ -31,8 +47,5 @@ export async function * ls (cid: CID, blockstore: GetStore, options: Partial<LsO
3147
throw new NotADirectoryError()
3248
}
3349

34-
yield * result.content({
35-
offset: options.offset,
36-
length: options.length
37-
})
50+
yield * result.content(options)
3851
}

‎packages/unixfs/src/index.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ import type { AbortOptions } from '@libp2p/interface'
8181
import type { Filter } from '@libp2p/utils/filters'
8282
import type { Blockstore } from 'interface-blockstore'
8383
import type { Mtime, UnixFS as IPFSUnixFS } from 'ipfs-unixfs'
84-
import type { ExporterProgressEvents, UnixFSEntry } from 'ipfs-unixfs-exporter'
84+
import type { ExporterProgressEvents, UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'
8585
import type { ByteStream, DirectoryCandidate, ImportCandidateStream, ImporterOptions, ImporterProgressEvents, ImportResult, ImportContent } from 'ipfs-unixfs-importer'
8686
import type { CID, Version } from 'multiformats/cid'
8787
import type { ProgressOptions } from 'progress-events'
@@ -210,6 +210,14 @@ export interface LsOptions extends AbortOptions, ProgressOptions<GetEvents> {
210210
* missing from the local store. (default: false)
211211
*/
212212
offline?: boolean
213+
214+
/**
215+
* If true, including UnixFS metadata in the output - nb. this will resolve
216+
* the root node of every encountered filesystem entry
217+
*
218+
* @default true
219+
*/
220+
extended?: boolean
213221
}
214222

215223
/**
@@ -644,6 +652,7 @@ export interface UnixFS {
644652
* ```
645653
*/
646654
ls(cid: CID, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry>
655+
ls(cid: CID, options: Partial<LsOptions> & { extended: false }): AsyncIterable<UnixFSBasicEntry>
647656

648657
/**
649658
* Make a new directory under an existing directory.

‎packages/unixfs/test/ls.spec.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,4 +133,59 @@ describe('ls', () => {
133133
}))).to.eventually.be.rejected
134134
.with.property('name', 'NotFoundError')
135135
})
136+
137+
it('should list a basic entry', async () => {
138+
const cid = await fs.addBytes(smallFile, {
139+
rawLeaves: false
140+
})
141+
142+
const files = await all(fs.ls(cid))
143+
144+
expect(files).to.have.nested.property('[0].type')
145+
expect(files).to.have.nested.property('[0].content')
146+
147+
const basicFiles = await all(fs.ls(cid, {
148+
extended: false
149+
}))
150+
151+
expect(basicFiles).to.not.have.nested.property('[0].type')
152+
expect(basicFiles).to.not.have.nested.property('[0].content')
153+
})
154+
155+
it('lists basic files in a directory', async () => {
156+
const path = 'path'
157+
const data = Uint8Array.from([0, 1, 2, 3])
158+
const fileCid = await fs.addBytes(data, {
159+
rawLeaves: false
160+
})
161+
const dirCid = await fs.cp(fileCid, emptyDirCid, path)
162+
163+
const files = await all(fs.ls(dirCid))
164+
165+
expect(files).to.have.nested.property('[0].type')
166+
expect(files).to.have.nested.property('[0].content')
167+
168+
const basicFiles = await all(fs.ls(dirCid, {
169+
extended: false
170+
}))
171+
172+
expect(basicFiles).to.not.have.nested.property('[0].type')
173+
expect(basicFiles).to.not.have.nested.property('[0].content')
174+
})
175+
176+
it('lists basic contents of a sharded directory', async () => {
177+
const shardedDirCid = await createShardedDirectory(blockstore)
178+
179+
const files = await all(fs.ls(shardedDirCid))
180+
181+
expect(files).to.have.nested.property('[0].type')
182+
expect(files).to.have.nested.property('[0].content')
183+
184+
const basicFiles = await all(fs.ls(shardedDirCid, {
185+
extended: false
186+
}))
187+
188+
expect(basicFiles).to.not.have.nested.property('[0].type')
189+
expect(basicFiles).to.not.have.nested.property('[0].content')
190+
})
136191
})

0 commit comments

Comments
 (0)