Skip to content

Commit b0033ca

Browse files
achingbrainachingbrain
authored
feat!: return cid/name/path from ls (#929)
Instead of reading blocks for each directory entry, just return the CID/name/path of the entry. BREAKING CHANGE: metadata is no longer returned from fs.ls - use fs.stat or similar to obtain it
1 parent 362f0b2 commit b0033ca

16 files changed

Lines changed: 75 additions & 204 deletions

File tree

‎packages/mfs/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,9 @@
5252
"interface-blockstore": "^6.0.1",
5353
"interface-datastore": "^9.0.2",
5454
"ipfs-unixfs": "^12.0.0",
55-
"ipfs-unixfs-exporter": "^14.0.1",
55+
"ipfs-unixfs-exporter": "^15.0.2",
5656
"ipfs-unixfs-importer": "^16.0.1",
57+
"it-map": "^3.1.4",
5758
"multiformats": "^13.4.1"
5859
},
5960
"devDependencies": {

‎packages/mfs/src/index.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import type { ComponentLogger } from '@libp2p/interface'
3434
import type { Blockstore } from 'interface-blockstore'
3535
import type { Datastore } from 'interface-datastore'
3636
import type { Mtime } from 'ipfs-unixfs'
37-
import type { UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'
37+
import type { UnixFSDirectoryEntry } from 'ipfs-unixfs-exporter'
3838
import type { ByteStream } from 'ipfs-unixfs-importer'
3939

4040
export interface MFSComponents {
@@ -171,8 +171,7 @@ export interface MFS {
171171
* }
172172
* ```
173173
*/
174-
ls(path?: string, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry>
175-
ls(path: string, options: Partial<LsOptions> & { extended: false }): AsyncIterable<UnixFSBasicEntry>
174+
ls(path?: string, options?: Partial<LsOptions>): AsyncIterable<UnixFSDirectoryEntry>
176175

177176
/**
178177
* Make a new directory in your MFS.

‎packages/mfs/src/mfs.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@ import { unixfs } from '@helia/unixfs'
22
import { AlreadyExistsError, DoesNotExistError, InvalidParametersError, NotADirectoryError } from '@helia/unixfs/errors'
33
import { Key } from 'interface-datastore'
44
import { UnixFS as IPFSUnixFS } from 'ipfs-unixfs'
5+
import map from 'it-map'
56
import { CID } from 'multiformats/cid'
67
import { basename } from './utils/basename.js'
78
import type { MFSComponents, MFSInit, MFS as MFSInterface, MkdirOptions, RmOptions, WriteOptions } from './index.js'
89
import type { CatOptions, ChmodOptions, CpOptions, LsOptions, StatOptions, TouchOptions, UnixFS, FileStats, DirectoryStats, RawStats, ExtendedStatOptions, ExtendedFileStats, ExtendedDirectoryStats, ExtendedRawStats } from '@helia/unixfs'
910
import type { AbortOptions, Logger } from '@libp2p/interface'
10-
import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
11+
import type { UnixFSDirectoryEntry } from 'ipfs-unixfs-exporter'
1112
import type { ByteStream } from 'ipfs-unixfs-importer'
1213

1314
interface PathEntry {
@@ -140,16 +141,30 @@ export class MFS implements MFSInterface {
140141
this.root = await this.#persistPath(trail, options)
141142
}
142143

143-
async * ls (path?: string, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry> {
144+
async * ls (path?: string, options?: Partial<LsOptions>): AsyncIterable<UnixFSDirectoryEntry> {
144145
const root = await this.#getRootCID()
145146

146147
if (options?.path != null) {
147148
path = `${path}/${options.path}`
148149
}
149150

150-
yield * this.unixfs.ls(root, {
151+
const rootString = root.toString()
152+
153+
yield * map(this.unixfs.ls(root, {
151154
...options,
152155
path
156+
}), (file) => {
157+
// remove CID from start of path
158+
let filePath = file.path.split('/').slice(1).join('/')
159+
160+
if (filePath.startsWith(rootString)) {
161+
filePath = filePath.substring(0, rootString.length)
162+
}
163+
164+
return {
165+
...file,
166+
path: `${path === '/' ? '' : path}/${filePath}`
167+
}
153168
})
154169
}
155170

‎packages/mfs/test/ls.spec.ts‎

Lines changed: 8 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,7 @@ describe('ls', () => {
3939
expect(files).to.have.lengthOf(1).and.to.containSubset([{
4040
cid: fileStat.cid,
4141
name: fileName,
42-
size: BigInt(data.byteLength),
43-
type: 'raw'
42+
path: filePath
4443
}])
4544
})
4645

@@ -59,8 +58,7 @@ describe('ls', () => {
5958
expect(files).to.have.lengthOf(1).and.to.containSubset([{
6059
cid: fileStat.cid,
6160
name: fileName,
62-
size: BigInt(data.byteLength),
63-
type: 'raw'
61+
path: filePath
6462
}])
6563
})
6664

@@ -121,8 +119,8 @@ describe('ls', () => {
121119

122120
expect(files).to.have.lengthOf(1).and.to.containSubset([{
123121
cid: fileStat.cid,
124-
size: BigInt(data.byteLength),
125-
type: 'raw'
122+
name: fileName,
123+
path: filePath
126124
}])
127125
})
128126

@@ -135,10 +133,12 @@ describe('ls', () => {
135133

136134
expect(files.length).to.equal(fileCount)
137135

138-
files.forEach(file => {
136+
for (const entry of files) {
137+
const file = await fs.stat(entry.path)
138+
139139
// should be a file
140140
expect(file.type).to.equal('raw')
141-
})
141+
}
142142
})
143143

144144
it('lists a file inside a sharded directory directly', async () => {
@@ -179,67 +179,4 @@ describe('ls', () => {
179179
expect(files.length).to.equal(1)
180180
expect(files.filter(file => file.name === fileName)).to.be.ok()
181181
})
182-
183-
it('should list a basic entry', async () => {
184-
const filePath = '/foo.txt'
185-
186-
await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), filePath, {
187-
rawLeaves: false,
188-
force: true
189-
})
190-
191-
const files = await all(fs.ls(filePath))
192-
193-
expect(files).to.have.nested.property('[0].type')
194-
expect(files).to.have.nested.property('[0].content')
195-
196-
const basicFiles = await all(fs.ls(filePath, {
197-
extended: false
198-
}))
199-
200-
expect(basicFiles).to.not.have.nested.property('[0].type')
201-
expect(basicFiles).to.not.have.nested.property('[0].content')
202-
})
203-
204-
it('lists basic files in a directory', async () => {
205-
const dirName = 'bar'
206-
const dirPath = `/${dirName}`
207-
const fileName = 'foo.txt'
208-
const filePath = `${dirPath}/${fileName}`
209-
210-
await fs.writeBytes(Uint8Array.from([0, 1, 2, 3]), filePath, {
211-
rawLeaves: false,
212-
force: true
213-
})
214-
215-
const files = await all(fs.ls(dirPath))
216-
217-
expect(files).to.have.nested.property('[0].type')
218-
expect(files).to.have.nested.property('[0].content')
219-
220-
const basicFiles = await all(fs.ls(dirPath, {
221-
extended: false
222-
}))
223-
224-
expect(basicFiles).to.not.have.nested.property('[0].type')
225-
expect(basicFiles).to.not.have.nested.property('[0].content')
226-
})
227-
228-
it('lists basic contents of a sharded directory', async () => {
229-
const shardedDirPath = '/sharded-dir'
230-
const shardedDirCid = await createShardedDirectory(blockstore)
231-
await fs.cp(shardedDirCid, shardedDirPath)
232-
233-
const files = await all(fs.ls(shardedDirPath))
234-
235-
expect(files).to.have.nested.property('[0].type')
236-
expect(files).to.have.nested.property('[0].content')
237-
238-
const basicFiles = await all(fs.ls(shardedDirPath, {
239-
extended: false
240-
}))
241-
242-
expect(basicFiles).to.not.have.nested.property('[0].type')
243-
expect(basicFiles).to.not.have.nested.property('[0].content')
244-
})
245182
})

‎packages/mfs/test/touch.spec.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,9 @@ describe('touch', () => {
142142
// no bigint support
143143
.that.satisfies((s: bigint) => s > seconds)
144144

145-
for await (const file of fs.ls(shardedDirPath)) {
145+
for await (const entry of fs.ls(shardedDirPath)) {
146+
const file = await fs.stat(entry.path)
147+
146148
expect(file).to.have.nested.property('unixfs.mtime.secs')
147149
// no bigint support
148150
.that.satisfies((s: bigint) => s > seconds)

‎packages/unixfs/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@
7676
"hamt-sharding": "^3.0.6",
7777
"interface-blockstore": "^6.0.1",
7878
"ipfs-unixfs": "^12.0.0",
79-
"ipfs-unixfs-exporter": "^14.0.1",
79+
"ipfs-unixfs-exporter": "^15.0.2",
8080
"ipfs-unixfs-importer": "^16.0.1",
8181
"it-all": "^3.0.9",
8282
"it-first": "^3.0.9",

‎packages/unixfs/src/commands/chmod.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import * as dagPB from '@ipld/dag-pb'
22
import { logger } from '@libp2p/logger'
33
import { UnixFS } from 'ipfs-unixfs'
4-
import { recursive } from 'ipfs-unixfs-exporter'
4+
import { exporter, recursive } from 'ipfs-unixfs-exporter'
55
import { importer } from 'ipfs-unixfs-importer'
66
import last from 'it-last'
77
import { pipe } from 'it-pipe'
@@ -31,13 +31,14 @@ export async function chmod (cid: CID, mode: number, blockstore: PutStore & GetS
3131
for await (const entry of recursive(resolved.cid, blockstore, options)) {
3232
let metadata: UnixFS
3333
let links: PBLink[] = []
34+
const file = await exporter(entry.cid, blockstore, options)
3435

35-
if (entry.type === 'raw') {
36+
if (file.type === 'raw') {
3637
// convert to UnixFS
37-
metadata = new UnixFS({ type: 'file', data: entry.node })
38-
} else if (entry.type === 'file' || entry.type === 'directory') {
39-
metadata = entry.unixfs
40-
links = entry.node.Links
38+
metadata = new UnixFS({ type: 'file', data: file.node })
39+
} else if (file.type === 'file' || file.type === 'directory') {
40+
metadata = file.unixfs
41+
links = file.node.Links
4142
} else {
4243
throw new NotUnixFSError()
4344
}

‎packages/unixfs/src/commands/ls.ts‎

Lines changed: 6 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,18 @@
11
import { exporter } from 'ipfs-unixfs-exporter'
2-
import { NoContentError, NotADirectoryError } from '../errors.js'
32
import { resolve } from './utils/resolve.js'
43
import type { LsOptions } from '../index.js'
54
import type { GetStore } from '../unixfs.js'
6-
import type { UnixFSEntry, UnixFSBasicEntry } from 'ipfs-unixfs-exporter'
5+
import type { UnixFSDirectoryEntry } from 'ipfs-unixfs-exporter'
76
import type { CID } from 'multiformats/cid'
87

9-
export function ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions & { extended: false }>): AsyncIterable<UnixFSBasicEntry>
10-
export function ls (cid: CID, blockstore: GetStore, options?: Partial<LsOptions>): AsyncIterable<UnixFSEntry>
11-
export async function * ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions> = {}): AsyncIterable<any> {
8+
export async function * ls (cid: CID, blockstore: GetStore, options: Partial<LsOptions> = {}): AsyncIterable<UnixFSDirectoryEntry> {
129
const resolved = await resolve(cid, options.path, blockstore, options)
13-
const result = await exporter(resolved.cid, blockstore, {
14-
...options,
15-
extended: true
16-
})
17-
18-
if (result.type === 'file' || result.type === 'raw') {
19-
if (options.extended === false) {
20-
const basic: UnixFSBasicEntry = {
21-
name: result.name,
22-
path: result.path,
23-
cid: result.cid
24-
}
25-
26-
yield basic
27-
} else {
28-
yield result
29-
}
10+
const result = await exporter(resolved.cid, blockstore, options)
3011

12+
if (result.type === 'directory') {
13+
yield * result.entries(options)
3114
return
3215
}
3316

34-
if (result.content == null) {
35-
throw new NoContentError()
36-
}
37-
38-
if (result.type !== 'directory') {
39-
throw new NotADirectoryError()
40-
}
41-
42-
yield * result.content(options)
17+
yield result
4318
}

‎packages/unixfs/src/commands/touch.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import * as dagPB from '@ipld/dag-pb'
22
import { logger } from '@libp2p/logger'
33
import { UnixFS } from 'ipfs-unixfs'
4-
import { recursive } from 'ipfs-unixfs-exporter'
4+
import { exporter, recursive } from 'ipfs-unixfs-exporter'
55
import { importer } from 'ipfs-unixfs-importer'
66
import last from 'it-last'
77
import { pipe } from 'it-pipe'
@@ -35,13 +35,14 @@ export async function touch (cid: CID, blockstore: GetStore & PutStore, options:
3535
for await (const entry of recursive(resolved.cid, blockstore)) {
3636
let metadata: UnixFS
3737
let links: PBLink[]
38+
const file = await exporter(entry.cid, blockstore, options)
3839

39-
if (entry.type === 'raw') {
40-
metadata = new UnixFS({ data: entry.node })
40+
if (file.type === 'raw') {
41+
metadata = new UnixFS({ data: file.node })
4142
links = []
42-
} else if (entry.type === 'file' || entry.type === 'directory') {
43-
metadata = entry.unixfs
44-
links = entry.node.Links
43+
} else if (file.type === 'file' || file.type === 'directory') {
44+
metadata = file.unixfs
45+
links = file.node.Links
4546
} else {
4647
throw new NotUnixFSError()
4748
}

‎packages/unixfs/src/commands/utils/remove-link.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,19 +141,21 @@ const convertToFlatDirectory = async (parent: Directory, blockstore: PutStore &
141141
const rootNode: PBNode = {
142142
Links: []
143143
}
144-
const dir = await exporter(parent.cid, blockstore)
144+
const dir = await exporter(parent.cid, blockstore, options)
145145

146146
if (dir.type !== 'directory') {
147147
throw new Error('Unexpected node type')
148148
}
149149

150-
for await (const entry of dir.content()) {
150+
for await (const entry of dir.entries()) {
151151
let tsize = 0
152152

153-
if (entry.node instanceof Uint8Array) {
154-
tsize = entry.node.byteLength
153+
const file = await exporter(entry.cid, blockstore, options)
154+
155+
if (file.node instanceof Uint8Array) {
156+
tsize = file.node.byteLength
155157
} else {
156-
tsize = dagPB.encode(entry.node).length
158+
tsize = dagPB.encode(file.node).length
157159
}
158160

159161
rootNode.Links.push({

0 commit comments

Comments
 (0)