feat: add globSource and urlSource (#53)

Ports `globSource` and `urlSource` from `ipfs-utils` for use with `unixfs.add` and `unixfs.addAll`.

Author: achingbrain

packages/unixfs/.aegir.js

@@ -0,0 +1,23 @@
+import EchoServer from 'aegir/echo-server'
+import { format } from 'iso-url'
+
+export default {
+  test: {
+    async before (options) {
+      let echoServer = new EchoServer()
+      await echoServer.start()
+      const { address, port } = echoServer.server.address()
+      let hostname = address
+      if(options.runner === 'react-native-android') {
+        hostname = '10.0.2.2'
+      }
+      return {
+        echoServer,
+        env: { ECHO_SERVER : format({ protocol: 'http:', hostname, port })}
+      }
+    },
+    async after (options, before) {
+      await before.echoServer.stop()
+    }
+  }
+}

packages/unixfs/package.json

@@ -165,6 +165,7 @@
     "ipfs-unixfs": "^11.0.0",
     "ipfs-unixfs-exporter": "^13.1.0",
     "ipfs-unixfs-importer": "^15.1.0",
+    "it-glob": "^2.0.4",
     "it-last": "^3.0.1",
     "it-pipe": "^3.0.1",
     "merge-options": "^3.0.4",
@@ -176,13 +177,22 @@
     "aegir": "^39.0.8",
     "blockstore-core": "^4.0.1",
     "delay": "^6.0.0",
+    "iso-url": "^1.2.1",
     "it-all": "^3.0.1",
     "it-drain": "^3.0.1",
     "it-first": "^3.0.1",
     "it-to-buffer": "^4.0.1",
-    "uint8arrays": "^4.0.3"
+    "uint8arrays": "^4.0.3",
+    "wherearewe": "^2.0.1"
   },
   "typedoc": {
     "entryPoint": "./src/index.ts"
+  },
+  "browser": {
+    "./dist/utils/glob-source.js": false,
+    "node:fs": false,
+    "node:fs/promises": false,
+    "node:path": false,
+    "node:url": false
   }
 }

packages/unixfs/src/index.ts

@@ -29,6 +29,18 @@
  *   console.info(entry)
  * }
  * ```
+ *
+ * @example
+ *
+ * Recursively adding a directory (Node.js-compatibly environments only):
+ *
+ * ```typescript
+ * import { globSource } from '@helia/unixfs'
+ *
+ * for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-pattern'))) {
+ *   console.info(entry)
+ * }
+ * ```
  */
 
 import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js'
@@ -607,3 +619,6 @@ class DefaultUnixFS implements UnixFS {
 export function unixfs (helia: { blockstore: Blocks }): UnixFS {
   return new DefaultUnixFS(helia)
 }
+
+export { globSource } from './utils/glob-source.js'
+export { urlSource } from './utils/url-source.js'

packages/unixfs/src/utils/glob-source.ts

@@ -0,0 +1,92 @@
+import fs from 'node:fs'
+import fsp from 'node:fs/promises'
+import Path from 'node:path'
+import glob from 'it-glob'
+import { InvalidParametersError } from '../errors.js'
+import type { MtimeLike } from 'ipfs-unixfs'
+import type { ImportCandidateStream } from 'ipfs-unixfs-importer'
+
+export interface GlobSourceOptions {
+  /**
+   * Include .dot files in matched paths
+   */
+  hidden?: boolean
+
+  /**
+   * follow symlinks
+   */
+  followSymlinks?: boolean
+
+  /**
+   * Preserve mode
+   */
+  preserveMode?: boolean
+
+  /**
+   * Preserve mtime
+   */
+  preserveMtime?: boolean
+
+  /**
+   * mode to use - if preserveMode is true this will be ignored
+   */
+  mode?: number
+
+  /**
+   * mtime to use - if preserveMtime is true this will be ignored
+   */
+  mtime?: MtimeLike
+}
+
+export interface GlobSourceResult {
+  path: string
+  content: AsyncIterable<Uint8Array> | undefined
+  mode: number | undefined
+  mtime: MtimeLike | undefined
+}
+
+/**
+ * Create an async iterator that yields paths that match requested glob pattern
+ */
+export async function * globSource (cwd: string, pattern: string, options: GlobSourceOptions = {}): ImportCandidateStream {
+  if (typeof pattern !== 'string') {
+    throw new InvalidParametersError('Pattern must be a string')
+  }
+
+  if (!Path.isAbsolute(cwd)) {
+    cwd = Path.resolve(process.cwd(), cwd)
+  }
+
+  const globOptions = Object.assign({}, {
+    nodir: false,
+    realpath: false,
+    absolute: true,
+    dot: Boolean(options.hidden),
+    follow: options.followSymlinks != null ? options.followSymlinks : true
+  })
+
+  for await (const p of glob(cwd, pattern, globOptions)) {
+    const stat = await fsp.stat(p)
+
+    let mode = options.mode
+
+    if (options.preserveMode === true) {
+      mode = stat.mode
+    }
+
+    let mtime = options.mtime
+
+    if (options.preserveMtime === true) {
+      mtime = stat.mtime
+    }
+
+    yield {
+      path: toPosix(p.replace(cwd, '')),
+      content: stat.isFile() ? fs.createReadStream(p) : undefined,
+      mode,
+      mtime
+    }
+  }
+}
+
+const toPosix = (path: string): string => path.replace(/\\/g, '/')

packages/unixfs/src/utils/url-source.ts

@@ -0,0 +1,35 @@
+import { UnknownError } from '../errors.js'
+import type { FileCandidate } from 'ipfs-unixfs-importer'
+
+export function urlSource (url: URL, options?: RequestInit): FileCandidate<AsyncGenerator<Uint8Array, void, unknown>> {
+  return {
+    path: decodeURIComponent(new URL(url).pathname.split('/').pop() ?? ''),
+    content: readURLContent(url, options)
+  }
+}
+
+async function * readURLContent (url: URL, options?: RequestInit): AsyncGenerator<Uint8Array, void, unknown> {
+  const response = await globalThis.fetch(url, options)
+
+  if (response.body == null) {
+    throw new UnknownError('HTTP response did not have a body')
+  }
+
+  const reader = response.body.getReader()
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read()
+
+      if (done) {
+        return
+      }
+
+      if (value != null) {
+        yield value
+      }
+    }
+  } finally {
+    reader.releaseLock()
+  }
+}

packages/unixfs/test/add.spec.ts

@@ -3,7 +3,9 @@
 import { expect } from 'aegir/chai'
 import { MemoryBlockstore } from 'blockstore-core'
 import all from 'it-all'
-import { unixfs, type UnixFS } from '../src/index.js'
+import last from 'it-last'
+import { isNode } from 'wherearewe'
+import { globSource, unixfs, urlSource, type UnixFS } from '../src/index.js'
 import type { Blockstore } from 'interface-blockstore'
 
 describe('addAll', () => {
@@ -29,6 +31,16 @@ describe('addAll', () => {
     expect(output[0].cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu')
     expect(output[1].cid.toString()).to.equal('bafkreidmuy2n45xj3cdknzprtzo2uvgm3hak6mzy5sllxty457agsftd34')
   })
+
+  it('recursively adds a directory', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const res = await last(fs.addAll(globSource('./test/fixtures', 'files/**/*')))
+
+    expect(res?.cid.toString()).to.equal('bafybeievhllpjjjbyg53g74wcl5hckdccjjj7zgtexqcacjegoduegnkyu')
+  })
 })
 
 describe('addBytes', () => {
@@ -82,6 +94,12 @@ describe('addFile', () => {
 
     expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu')
   })
+
+  it('adds a file from a URL', async () => {
+    const cid = await fs.addFile(urlSource(new URL(`${process.env.ECHO_SERVER}/download?data=hello-world`)))
+
+    expect(cid.toString()).to.equal('bafkreifpuj5ujvb3aku75ja5cphnylsac3h47b6f3p4zbzmtm2nkrtrinu')
+  })
 })
 
 describe('addDirectory', () => {

packages/unixfs/test/fixtures/files/another-dir/another-nested-dir/other.txt

@@ -0,0 +1 @@
+hello