diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 5000efbf..f11662da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) + +### Documentation + +* convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) + ## [ipfs-unixfs-exporter-v13.6.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.3...ipfs-unixfs-exporter-13.6.4) (2025-06-18) ### Dependencies diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 127fd8d8..2110db9e 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.4", + "version": "13.6.5", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index a48fe9bb..67326ec4 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -31,6 +31,7 @@ import { exporter, recursive } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' +import type { UnixFSType } from 'ipfs-unixfs' import type { Chunker } from 'ipfs-unixfs-importer/chunker' import type { FileLayout } from 'ipfs-unixfs-importer/layout' @@ -46,7 +47,7 @@ describe('exporter', () => { smallFile = uint8ArrayConcat(await all(randomBytes(200))) }) - async function dagPut (options: { type?: string, content?: Uint8Array, links?: dagPb.PBLink[] } = {}): Promise<{ file: UnixFS, node: PBNode, cid: CID }> { + async function dagPut (options: { type?: UnixFSType, content?: Uint8Array, links?: dagPb.PBLink[] } = {}): Promise<{ file: UnixFS, node: PBNode, cid: CID }> { options.type = options.type ?? 'file' options.content = options.content ?? Uint8Array.from([0x01, 0x02, 0x03]) options.links = options.links ?? [] diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index bbcf3bc2..98e3b825 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,15 @@ +## [ipfs-unixfs-importer-v15.4.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.4...ipfs-unixfs-importer-15.4.0) (2025-06-18) + +### Features + +* enable custom file/directory builders ([#413](https://github.com/ipfs/js-ipfs-unixfs/issues/413)) ([d06d9fe](https://github.com/ipfs/js-ipfs-unixfs/commit/d06d9fe5aa4c0e7f82f3265bf09b0db064d2b563)) + +## [ipfs-unixfs-importer-v15.3.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.3...ipfs-unixfs-importer-15.3.4) (2025-06-18) + +### Documentation + +* convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) + ## [ipfs-unixfs-importer-v15.3.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.2...ipfs-unixfs-importer-15.3.3) (2025-06-18) ### Dependencies diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 925ade49..ac50610b 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.3", + "version": "15.4.0", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts index a29675f6..06996f46 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts @@ -9,7 +9,11 @@ export interface DirBuilderOptions { signal?: AbortSignal } -export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { +export interface DirBuilder { + (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise +} + +export const defaultDirBuilder: DirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { const unixfs = new UnixFS({ type: 'directory', mtime: dir.mtime, diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 41862bce..db359606 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -30,7 +30,7 @@ async function * buildFileBatch (file: File, blockstore: WritableStorage, option } continue - } else if (count === 1 && (previous != null)) { + } else if (count === 1 && previous != null) { // we have the second block of a multiple block import so yield the first yield { ...previous, @@ -131,7 +131,7 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return true } - if ((leaf.unixfs != null) && (leaf.unixfs.data == null) && leaf.unixfs.fileSize() > 0n) { + if (leaf.unixfs != null && leaf.unixfs.data == null && leaf.unixfs.fileSize() > 0n) { return true } @@ -189,10 +189,17 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return reducer } +export interface FileBuilder { + (file: File, blockstore: WritableStorage, options: FileBuilderOptions): Promise +} + export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions { layout: FileLayout } -export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { - return options.layout(buildFileBatch(file, block, options), reduce(file, block, options)) +export const defaultFileBuilder: FileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { + return options.layout( + buildFileBatch(file, block, options), + reduce(file, block, options) + ) } diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 33a032b7..475c3515 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,9 +1,9 @@ import { CustomProgressEvent } from 'progress-events' import { InvalidContentError } from '../errors.js' -import { dirBuilder } from './dir.js' -import { fileBuilder } from './file.js' -import type { DirBuilderOptions } from './dir.js' -import type { FileBuilderOptions } from './file.js' +import { defaultDirBuilder } from './dir.js' +import { defaultFileBuilder } from './file.js' +import type { DirBuilder, DirBuilderOptions } from './dir.js' +import type { FileBuilder, FileBuilderOptions } from './file.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.js' @@ -45,11 +45,11 @@ function contentAsAsyncIterable (content: Uint8Array | AsyncIterable if (content instanceof Uint8Array) { return (async function * () { yield content - }()) + })() } else if (isIterable(content)) { return (async function * () { yield * content - }()) + })() } else if (isAsyncIterable(content)) { return content } @@ -64,6 +64,8 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions chunker: Chunker chunkValidator: ChunkValidator wrapWithDirectory: boolean + dirBuilder?: DirBuilder + fileBuilder?: FileBuilder } export type ImporterSourceStream = AsyncIterable | Iterable @@ -109,6 +111,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const fileBuilder = options.fileBuilder ?? defaultFileBuilder + yield async () => fileBuilder(file, blockstore, options) } else if (entry.path != null) { const dir: Directory = { @@ -118,6 +122,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const dirBuilder = options.dirBuilder ?? defaultDirBuilder + yield async () => dirBuilder(dir, blockstore, options) } else { throw new Error('Import candidate must have content or path or both') diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 84e22564..2f00975f 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -73,7 +73,8 @@ import { balanced } from './layout/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { Chunker } from './chunker/index.js' import type { BufferImportProgressEvents } from './dag-builder/buffer-importer.js' -import type { ReducerProgressEvents } from './dag-builder/file.js' +import type { DirBuilder } from './dag-builder/dir.js' +import type { FileBuilder, ReducerProgressEvents } from './dag-builder/file.js' import type { DAGBuilder, DagBuilderProgressEvents } from './dag-builder/index.js' import type { ChunkValidator } from './dag-builder/validate-chunks.js' import type { FileLayout } from './layout/index.js' @@ -276,6 +277,20 @@ export interface ImporterOptions extends ProgressOptions * `Error` */ chunkValidator?: ChunkValidator + + /** + * This option can be used to override how a directory IPLD node is built. + * + * This function takes a `Directory` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + dirBuilder?: DirBuilder + + /** + * This option can be used to override how a file IPLD node is built. + * + * This function takes a `File` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + fileBuilder?: FileBuilder } export type ImportCandidateStream = AsyncIterable | Iterable @@ -342,7 +357,9 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri blockWriteConcurrency, reduceSingleLeafToSelf, cidVersion, - onProgress: options.onProgress + onProgress: options.onProgress, + dirBuilder: options.dirBuilder, + fileBuilder: options.fileBuilder }) const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({ wrapWithDirectory, diff --git a/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts new file mode 100644 index 00000000..605e26fc --- /dev/null +++ b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts @@ -0,0 +1,40 @@ +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { defaultDirBuilder } from '../src/dag-builder/dir.js' +import { defaultFileBuilder } from '../src/dag-builder/file.js' +import { importer } from '../src/index.js' +import type { DirBuilder } from '../src/dag-builder/dir.js' +import type { FileBuilder } from '../src/dag-builder/file.js' + +describe('CustomParamsDagBuilder', () => { + it('should build a dag with custom dir builder', async () => { + const counter = { dirCounter: 0, fileCounter: 0 } + const customDirBuilder: DirBuilder = async (...args) => { + counter.dirCounter++ + return defaultDirBuilder(...args) + } + + const customFileBuilder: FileBuilder = async (...args) => { + counter.fileCounter++ + return defaultFileBuilder(...args) + } + + const blockstore = new MemoryBlockstore() + const files = [] + for await (const file of importer([{ + path: './src/file.txt', + content: new Uint8Array( + 'hello world'.split('').map((char) => char.charCodeAt(0)) + ) + }, { + path: './src' + }], blockstore, { + dirBuilder: customDirBuilder, + fileBuilder: customFileBuilder + })) { + files.push(file) + } + + expect(counter.dirCounter).to.equal(1) + }) +}) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 5f20200d..f5930d07 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.4...ipfs-unixfs-11.2.5) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-v11.2.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.3...ipfs-unixfs-11.2.4) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index d8757440..aa23bcb0 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -135,7 +135,7 @@ file.mtime // undefined Object.prototype.hasOwnProperty.call(file, 'mtime') // false -const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) +const dir = new UnixFS({ type: 'directory', mtime: { secs: 5n } }) dir.mtime // { secs: Number, nsecs: Number } ``` diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 6a81edef..fbace349 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.4", + "version": "11.2.5", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", diff --git a/packages/ipfs-unixfs/src/index.ts b/packages/ipfs-unixfs/src/index.ts index a5a98ed9..04e03973 100644 --- a/packages/ipfs-unixfs/src/index.ts +++ b/packages/ipfs-unixfs/src/index.ts @@ -112,7 +112,7 @@ * * Object.prototype.hasOwnProperty.call(file, 'mtime') // false * - * const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) + * const dir = new UnixFS({ type: 'directory', mtime: { secs: 5n } }) * dir.mtime // { secs: Number, nsecs: Number } * ``` */ @@ -127,7 +127,9 @@ export interface Mtime { export type MtimeLike = Mtime | { Seconds: number, FractionalNanoseconds?: number } | [number, number] | Date -const types: Record = { +export type UnixFSType = 'raw' | 'directory' | 'file' | 'metadata' | 'symlink' | 'hamt-sharded-directory' + +const types: Record = { Raw: 'raw', Directory: 'directory', File: 'file', @@ -148,7 +150,7 @@ const DEFAULT_DIRECTORY_MODE = parseInt('0755', 8) const MAX_FANOUT = BigInt(1 << 10) export interface UnixFSOptions { - type?: string + type?: UnixFSType data?: Uint8Array blockSizes?: bigint[] hashType?: bigint diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.ts b/packages/ipfs-unixfs/test/unixfs-format.spec.ts index 15c76373..0aba16d3 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.ts +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.ts @@ -370,6 +370,7 @@ describe('unixfs-format', () => { try { // eslint-disable-next-line no-new new UnixFS({ + // @ts-expect-error invalid type type: 'bananas' }) } catch (err: any) {