From 3ea4423bb1564fa71491d7a753720408762a4395 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 12:16:24 +0000 Subject: [PATCH 01/42] chore(release): 15.2.6 [skip ci] ## [ipfs-unixfs-importer-v15.2.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.5...ipfs-unixfs-importer-15.2.6) (2024-09-13) ### Dependencies * **dev:** bump aegir from 42.2.11 to 44.1.1 ([#412](https://github.com/ipfs/js-ipfs-unixfs/issues/412)) ([f94d1ad](https://github.com/ipfs/js-ipfs-unixfs/commit/f94d1ad0a507a0b37dd601490bba22224192f5a3)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 99b97f89..199ca22d 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.2.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.5...ipfs-unixfs-importer-15.2.6) (2024-09-13) + +### Dependencies + +* **dev:** bump aegir from 42.2.11 to 44.1.1 ([#412](https://github.com/ipfs/js-ipfs-unixfs/issues/412)) ([f94d1ad](https://github.com/ipfs/js-ipfs-unixfs/commit/f94d1ad0a507a0b37dd601490bba22224192f5a3)) + ## ipfs-unixfs-importer [15.2.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.4...ipfs-unixfs-importer-15.2.5) (2024-04-05) diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 48795894..ee02f082 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.2.5", + "version": "15.2.6", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From c345042368f1537fe5c6e9d1df5bd180ebbf755e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 12:16:39 +0000 Subject: [PATCH 02/42] chore(release): 13.5.1 [skip ci] ## [ipfs-unixfs-exporter-v13.5.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.0...ipfs-unixfs-exporter-13.5.1) (2024-09-13) ### Bug Fixes * add sideEffects false to package.json to enable tree shaking ([#402](https://github.com/ipfs/js-ipfs-unixfs/issues/402)) ([aea58c4](https://github.com/ipfs/js-ipfs-unixfs/commit/aea58c40a4a2457ddf44454befa1eb25d4caa016)) ### Trivial Changes * rename master to main ([0cdfcd6](https://github.com/ipfs/js-ipfs-unixfs/commit/0cdfcd674513b21aab7e27b446a6f2181c9ba842)) ### Dependencies * **dev:** bump aegir from 42.2.11 to 44.1.1 ([#412](https://github.com/ipfs/js-ipfs-unixfs/issues/412)) ([f94d1ad](https://github.com/ipfs/js-ipfs-unixfs/commit/f94d1ad0a507a0b37dd601490bba22224192f5a3)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 14 ++++++++++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index db0ca47b..a71f6e6a 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,17 @@ +## [ipfs-unixfs-exporter-v13.5.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.0...ipfs-unixfs-exporter-13.5.1) (2024-09-13) + +### Bug Fixes + +* add sideEffects false to package.json to enable tree shaking ([#402](https://github.com/ipfs/js-ipfs-unixfs/issues/402)) ([aea58c4](https://github.com/ipfs/js-ipfs-unixfs/commit/aea58c40a4a2457ddf44454befa1eb25d4caa016)) + +### Trivial Changes + +* rename master to main ([0cdfcd6](https://github.com/ipfs/js-ipfs-unixfs/commit/0cdfcd674513b21aab7e27b446a6f2181c9ba842)) + +### Dependencies + +* **dev:** bump aegir from 42.2.11 to 44.1.1 ([#412](https://github.com/ipfs/js-ipfs-unixfs/issues/412)) ([f94d1ad](https://github.com/ipfs/js-ipfs-unixfs/commit/f94d1ad0a507a0b37dd601490bba22224192f5a3)) + ## ipfs-unixfs-exporter [13.5.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.4.0...ipfs-unixfs-exporter-13.5.0) (2024-02-02) diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 0f9c27f4..07942a86 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.5.0", + "version": "13.5.1", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 70145c7c1e78ede3c481100151a9491c639671be Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Fri, 13 Sep 2024 14:49:11 +0100 Subject: [PATCH 03/42] feat: add name property to errors (#414) Adds a `.name` property to all errors that can be used in a more ideomatic way than `.code`. --- packages/ipfs-unixfs-exporter/package.json | 1 - packages/ipfs-unixfs-exporter/src/errors.ts | 87 +++++++++++++++++++ packages/ipfs-unixfs-exporter/src/index.ts | 10 ++- .../src/resolvers/identity.ts | 4 +- .../src/resolvers/index.ts | 4 +- .../ipfs-unixfs-exporter/src/resolvers/raw.ts | 4 +- .../src/resolvers/unixfs-v1/content/file.ts | 14 +-- .../content/hamt-sharded-directory.ts | 8 +- .../src/resolvers/unixfs-v1/index.ts | 10 +-- .../src/utils/find-cid-in-shard.ts | 10 +-- .../src/utils/resolve-object-path.ts | 4 +- .../src/utils/validate-offset-and-length.ts | 10 +-- packages/ipfs-unixfs-importer/package.json | 1 - .../ipfs-unixfs-importer/src/chunker/rabin.ts | 8 +- .../src/dag-builder/index.ts | 6 +- .../src/dag-builder/validate-chunks.ts | 6 +- packages/ipfs-unixfs-importer/src/errors.ts | 54 ++++++++++++ packages/ipfs-unixfs-importer/src/index.ts | 8 +- packages/ipfs-unixfs/package.json | 1 - packages/ipfs-unixfs/src/errors.ts | 10 +++ packages/ipfs-unixfs/src/index.ts | 7 +- 21 files changed, 210 insertions(+), 57 deletions(-) create mode 100644 packages/ipfs-unixfs-exporter/src/errors.ts create mode 100644 packages/ipfs-unixfs-importer/src/errors.ts create mode 100644 packages/ipfs-unixfs/src/errors.ts diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 07942a86..f5faaa1e 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -140,7 +140,6 @@ "@ipld/dag-json": "^10.2.2", "@ipld/dag-pb": "^4.1.2", "@multiformats/murmur3": "^2.1.8", - "err-code": "^3.0.1", "hamt-sharding": "^3.0.6", "interface-blockstore": "^5.3.0", "ipfs-unixfs": "^11.0.0", diff --git a/packages/ipfs-unixfs-exporter/src/errors.ts b/packages/ipfs-unixfs-exporter/src/errors.ts new file mode 100644 index 00000000..33ff7ee2 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/errors.ts @@ -0,0 +1,87 @@ +export class BadPathError extends Error { + static name = 'BadPathError' + static code = 'ERR_BAD_PATH' + name = BadPathError.name + code = BadPathError.code + + constructor (message = 'Bad path') { + super(message) + } +} + +export class NotFoundError extends Error { + static name = 'NotFoundError' + static code = 'ERR_NOT_FOUND' + name = NotFoundError.name + code = NotFoundError.code + + constructor (message = 'Not found') { + super(message) + } +} + +export class NoResolverError extends Error { + static name = 'NoResolverError' + static code = 'ERR_NO_RESOLVER' + name = NoResolverError.name + code = NoResolverError.code + + constructor (message = 'No resolver') { + super(message) + } +} + +export class NotUnixFSError extends Error { + static name = 'NotUnixFSError' + static code = 'ERR_NOT_UNIXFS' + name = NotUnixFSError.name + code = NotUnixFSError.code + + constructor (message = 'Not UnixFS') { + super(message) + } +} + +export class OverReadError extends Error { + static name = 'OverReadError' + static code = 'ERR_OVER_READ' + name = OverReadError.name + code = OverReadError.code + + constructor (message = 'Over read') { + super(message) + } +} + +export class UnderReadError extends Error { + static name = 'UnderReadError' + static code = 'ERR_UNDER_READ' + name = UnderReadError.name + code = UnderReadError.code + + constructor (message = 'Under read') { + super(message) + } +} + +export class NoPropError extends Error { + static name = 'NoPropError' + static code = 'ERR_NO_PROP' + name = NoPropError.name + code = NoPropError.code + + constructor (message = 'No Property found') { + super(message) + } +} + +export class InvalidParametersError extends Error { + static name = 'InvalidParametersError' + static code = 'ERR_INVALID_PARAMS' + name = InvalidParametersError.name + code = InvalidParametersError.code + + constructor (message = 'Invalid parameters') { + super(message) + } +} diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index b1adc319..ead6279a 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -45,9 +45,9 @@ * ``` */ -import errCode from 'err-code' import last from 'it-last' import { CID } from 'multiformats/cid' +import { BadPathError, NotFoundError } from './errors.js' import resolve from './resolvers/index.js' import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' @@ -55,6 +55,8 @@ import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' import type { ProgressOptions, ProgressEvent } from 'progress-events' +export * from './errors.js' + export interface ExportProgress { /** * How many bytes of the file have been read @@ -361,7 +363,7 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str } } - throw errCode(new Error(`Unknown path type ${path}`), 'ERR_BAD_PATH') + throw new BadPathError(`Unknown path type ${path}`) } /** @@ -394,7 +396,7 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockstore, options) if (result.entry == null && result.next == null) { - throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND') + throw new NotFoundError(`Could not resolve ${path}`) } if (result.entry != null) { @@ -441,7 +443,7 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, const result = await last(walkPath(path, blockstore, options)) if (result == null) { - throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND') + throw new NotFoundError(`Could not resolve ${path}`) } return result diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/identity.ts b/packages/ipfs-unixfs-exporter/src/resolvers/identity.ts index 861e6dc7..0ce21223 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/identity.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/identity.ts @@ -1,6 +1,6 @@ -import errCode from 'err-code' import * as mh from 'multiformats/hashes/digest' import { CustomProgressEvent } from 'progress-events' +import { NotFoundError } from '../errors.js' import extractDataFromBlock from '../utils/extract-data-from-block.js' import validateOffsetAndLength from '../utils/validate-offset-and-length.js' import type { ExporterOptions, Resolver, ExportProgress } from '../index.js' @@ -28,7 +28,7 @@ const rawContent = (node: Uint8Array): ((options?: ExporterOptions) => AsyncGene const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { if (toResolve.length > 0) { - throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND') + throw new NotFoundError(`No link named ${path} found in raw node ${cid}`) } const buf = mh.decode(cid.multihash.bytes) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/index.ts index c314fa67..099b8444 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/index.ts @@ -1,10 +1,10 @@ import * as dagCbor from '@ipld/dag-cbor' import * as dagJson from '@ipld/dag-json' import * as dagPb from '@ipld/dag-pb' -import errCode from 'err-code' import * as json from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' +import { NoResolverError } from '../errors.js' import dagCborResolver from './dag-cbor.js' import dagJsonResolver from './dag-json.js' import identifyResolver from './identity.js' @@ -26,7 +26,7 @@ const resolve: Resolve = async (cid, name, path, toResolve, depth, blockstore, o const resolver = resolvers[cid.code] if (resolver == null) { - throw errCode(new Error(`No resolver for code ${cid.code}`), 'ERR_NO_RESOLVER') + throw new NoResolverError(`No resolver for code ${cid.code}`) } return resolver(cid, name, path, toResolve, resolve, depth, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts b/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts index 7b5d2429..4082e388 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/raw.ts @@ -1,5 +1,5 @@ -import errCode from 'err-code' import { CustomProgressEvent } from 'progress-events' +import { NotFoundError } from '../errors.js' import extractDataFromBlock from '../utils/extract-data-from-block.js' import validateOffsetAndLength from '../utils/validate-offset-and-length.js' import type { ExporterOptions, Resolver, ExportProgress } from '../index.js' @@ -27,7 +27,7 @@ const rawContent = (node: Uint8Array): ((options?: ExporterOptions) => AsyncGene const resolve: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { if (toResolve.length > 0) { - throw errCode(new Error(`No link named ${path} found in raw node ${cid}`), 'ERR_NOT_FOUND') + throw new NotFoundError(`No link named ${path} found in raw node ${cid}`) } const block = await blockstore.get(cid, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index f65a449a..d88e1a24 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -1,5 +1,4 @@ import * as dagPb from '@ipld/dag-pb' -import errCode from 'err-code' import { UnixFS } from 'ipfs-unixfs' import map from 'it-map' import parallel from 'it-parallel' @@ -8,6 +7,7 @@ import { type Pushable, pushable } from 'it-pushable' import * as raw from 'multiformats/codecs/raw' import PQueue from 'p-queue' import { CustomProgressEvent } from 'progress-events' +import { NotUnixFSError, OverReadError, UnderReadError } from '../../../errors.js' import extractDataFromBlock from '../../../utils/extract-data-from-block.js' import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js' import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage, ExportProgress, ExportWalk } from '../../../index.js' @@ -23,7 +23,7 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A } if (node.Data == null) { - throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('no data in PBNode') } let file: UnixFS @@ -31,7 +31,7 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A try { file = UnixFS.unmarshal(node.Data) } catch (err: any) { - throw errCode(err, 'ERR_NOT_UNIXFS') + throw new NotUnixFSError(err.message) } // might be a unixfs `raw` node or have data on intermediate nodes @@ -47,7 +47,7 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A const childOps: Array<{ link: dagPb.PBLink, blockStart: bigint }> = [] if (node.Links.length !== file.blockSizes.length) { - throw errCode(new Error('Inconsistent block sizes and dag links'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('Inconsistent block sizes and dag links') } for (let i = 0; i < node.Links.length; i++) { @@ -98,7 +98,7 @@ async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8A child = block break default: - queue.end(errCode(new Error(`Unsupported codec: ${link.Hash.code}`), 'ERR_NOT_UNIXFS')) + queue.end(new NotUnixFSError(`Unsupported codec: ${link.Hash.code}`)) return } @@ -171,7 +171,7 @@ const fileContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, if (read > wanted) { queue.end() - throw errCode(new Error('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect'), 'ERR_OVER_READ') + throw new OverReadError('Read too many bytes - the file size reported by the UnixFS data in the root node may be incorrect') } if (read === wanted) { @@ -188,7 +188,7 @@ const fileContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, } if (read < wanted) { - throw errCode(new Error('Traversed entire DAG but did not read enough bytes'), 'ERR_UNDER_READ') + throw new UnderReadError('Traversed entire DAG but did not read enough bytes') } } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index 1c482c68..52c2938d 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -1,10 +1,10 @@ import { decode, type PBNode } from '@ipld/dag-pb' -import errCode from 'err-code' import { UnixFS } from 'ipfs-unixfs' import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' +import { NotUnixFSError } from '../../../errors.js' import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { @@ -23,18 +23,18 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de const links = node.Links if (node.Data == null) { - throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('no data in PBNode') } let dir: UnixFS try { dir = UnixFS.unmarshal(node.Data) } catch (err: any) { - throw errCode(err, 'ERR_NOT_UNIXFS') + throw new NotUnixFSError(err.message) } if (dir.fanout == null) { - throw errCode(new Error('missing fanout'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('missing fanout') } const padLength = (dir.fanout - 1n).toString(16).length diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 148ac81f..2ed6c6a0 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -1,6 +1,6 @@ import { decode, type PBNode } from '@ipld/dag-pb' -import errCode from 'err-code' import { UnixFS } from 'ipfs-unixfs' +import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' @@ -39,14 +39,14 @@ const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, dep } if (node.Data == null) { - throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('no data in PBNode') } try { unixfs = UnixFS.unmarshal(node.Data) } catch (err: any) { // non-UnixFS dag-pb node? It could happen. - throw errCode(err, 'ERR_NOT_UNIXFS') + throw new NotUnixFSError(err.message) } if (path == null) { @@ -64,7 +64,7 @@ const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, dep } if (linkCid == null) { - throw errCode(new Error('file does not exist'), 'ERR_NOT_FOUND') + throw new NotFoundError('file does not exist') } // remove the path component we have resolved @@ -82,7 +82,7 @@ const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, dep const content = contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, blockstore) if (content == null) { - throw errCode(new Error('could not find content exporter'), 'ERR_NOT_FOUND') + throw new NotFoundError('could not find content exporter') } if (unixfs.isDirectory()) { diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts index 13346bf7..d3446ddd 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts @@ -1,8 +1,8 @@ import { decode, type PBLink, type PBNode } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' -import errCode from 'err-code' import { Bucket, type BucketPosition, createHAMT } from 'hamt-sharding' import { UnixFS } from 'ipfs-unixfs' +import { NotUnixFSError } from '../errors.js' import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js' import type { CID } from 'multiformats/cid' @@ -66,21 +66,21 @@ const toBucketPath = (position: BucketPosition): Array> const findShardCid = async (node: PBNode, name: string, blockstore: ReadableStorage, context?: ShardTraversalContext, options?: ExporterOptions): Promise => { if (context == null) { if (node.Data == null) { - throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('no data in PBNode') } let dir: UnixFS try { dir = UnixFS.unmarshal(node.Data) } catch (err: any) { - throw errCode(err, 'ERR_NOT_UNIXFS') + throw new NotUnixFSError(err.message) } if (dir.type !== 'hamt-sharded-directory') { - throw errCode(new Error('not a HAMT'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('not a HAMT') } if (dir.fanout == null) { - throw errCode(new Error('missing fanout'), 'ERR_NOT_UNIXFS') + throw new NotUnixFSError('missing fanout') } const rootBucket = createHAMT({ diff --git a/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts b/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts index addb7066..b17f1ebc 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/resolve-object-path.ts @@ -1,5 +1,5 @@ -import errCode from 'err-code' import { CID } from 'multiformats/cid' +import { NoPropError } from '../errors.js' import type { ResolveResult } from '../index.js' export function resolveObjectPath (object: any, block: Uint8Array, cid: CID, name: string, path: string, toResolve: string[], depth: number): ResolveResult { @@ -41,7 +41,7 @@ export function resolveObjectPath (object: any, block: Uint8Array, cid: CID, nam subObject = subObject[prop] } else { // cannot resolve further - throw errCode(new Error(`No property named ${prop} found in node ${cid}`), 'ERR_NO_PROP') + throw new NoPropError(`No property named ${prop} found in node ${cid}`) } } diff --git a/packages/ipfs-unixfs-exporter/src/utils/validate-offset-and-length.ts b/packages/ipfs-unixfs-exporter/src/utils/validate-offset-and-length.ts index da6d9427..ca3e073a 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/validate-offset-and-length.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/validate-offset-and-length.ts @@ -1,4 +1,4 @@ -import errCode from 'err-code' +import { InvalidParametersError } from '../errors.js' const validateOffsetAndLength = (size: number | bigint, offset: number | bigint = 0, length: number | bigint = size): { start: bigint, end: bigint } => { const fileSize = BigInt(size) @@ -14,19 +14,19 @@ const validateOffsetAndLength = (size: number | bigint, offset: number | bigint } if (start < 0n) { - throw errCode(new Error('Offset must be greater than or equal to 0'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Offset must be greater than or equal to 0') } if (start > fileSize) { - throw errCode(new Error('Offset must be less than the file size'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Offset must be less than the file size') } if (end < 0n) { - throw errCode(new Error('Length must be greater than or equal to 0'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Length must be greater than or equal to 0') } if (end > fileSize) { - throw errCode(new Error('Length must be less than the file size'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Length must be less than the file size') } return { diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index ee02f082..396b48df 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -162,7 +162,6 @@ "dependencies": { "@ipld/dag-pb": "^4.1.2", "@multiformats/murmur3": "^2.1.8", - "err-code": "^3.0.1", "hamt-sharding": "^3.0.6", "interface-blockstore": "^5.3.0", "interface-store": "^5.1.8", diff --git a/packages/ipfs-unixfs-importer/src/chunker/rabin.ts b/packages/ipfs-unixfs-importer/src/chunker/rabin.ts index a64defdb..d491f04e 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/rabin.ts +++ b/packages/ipfs-unixfs-importer/src/chunker/rabin.ts @@ -1,7 +1,7 @@ -import errcode from 'err-code' // @ts-expect-error no types import { create } from 'rabin-wasm' import { Uint8ArrayList } from 'uint8arraylist' +import { InvalidAvgChunkSizeError, InvalidChunkSizeError, InvalidMinChunkSizeError } from '../errors.js' import type { Chunker } from './index.js' const DEFAULT_MIN_CHUNK_SIZE = 262144 @@ -54,15 +54,15 @@ export const rabin = (options: RabinOptions = {}): Chunker => { if (isInvalidChunkSizes) { if (options.avgChunkSize != null) { - throw errcode(new Error('please specify a valid average chunk size number'), 'ERR_INVALID_AVG_CHUNK_SIZE') + throw new InvalidAvgChunkSizeError('please specify a valid average chunk size number') } - throw errcode(new Error('please specify valid numbers for (min|max|avg)ChunkSize'), 'ERR_INVALID_CHUNK_SIZE') + throw new InvalidChunkSizeError('please specify valid numbers for (min|max|avg)ChunkSize') } // validate min/max/avg in the same way as go if (min < 16) { - throw errcode(new Error('rabin min must be greater than 16'), 'ERR_INVALID_MIN_CHUNK_SIZE') + throw new InvalidMinChunkSizeError('rabin min must be greater than 16') } if (max < min) { diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 1559b86a..b9bfe8f6 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,5 +1,5 @@ -import errCode from 'err-code' import { CustomProgressEvent } from 'progress-events' +import { InvalidContentError } from '../errors.js' import { dirBuilder, type DirBuilderOptions } from './dir.js' import { fileBuilder, type FileBuilderOptions } from './file.js' import type { ChunkValidator } from './validate-chunks.js' @@ -52,10 +52,10 @@ function contentAsAsyncIterable (content: Uint8Array | AsyncIterable return content } } catch { - throw errCode(new Error('Content was invalid'), 'ERR_INVALID_CONTENT') + throw new InvalidContentError('Content was invalid') } - throw errCode(new Error('Content was invalid'), 'ERR_INVALID_CONTENT') + throw new InvalidContentError('Content was invalid') } export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions, ProgressOptions { diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.ts b/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.ts index d01e0628..c1e3ea01 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.ts @@ -1,5 +1,5 @@ -import errCode from 'err-code' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { InvalidContentError } from '../errors.js' export interface ChunkValidator { (source: AsyncIterable): AsyncIterable } @@ -7,7 +7,7 @@ export const defaultChunkValidator = (): ChunkValidator => { return async function * validateChunks (source) { for await (const content of source) { if (content.length === undefined) { - throw errCode(new Error('Content was invalid'), 'ERR_INVALID_CONTENT') + throw new InvalidContentError('Content was invalid') } if (typeof content === 'string' || content instanceof String) { @@ -17,7 +17,7 @@ export const defaultChunkValidator = (): ChunkValidator => { } else if (content instanceof Uint8Array) { yield content } else { - throw errCode(new Error('Content was invalid'), 'ERR_INVALID_CONTENT') + throw new InvalidContentError('Content was invalid') } } } diff --git a/packages/ipfs-unixfs-importer/src/errors.ts b/packages/ipfs-unixfs-importer/src/errors.ts new file mode 100644 index 00000000..a8facda8 --- /dev/null +++ b/packages/ipfs-unixfs-importer/src/errors.ts @@ -0,0 +1,54 @@ +export class InvalidParametersError extends Error { + static name = 'InvalidParametersError' + static code = 'ERR_INVALID_PARAMS' + name = InvalidParametersError.name + code = InvalidParametersError.code + + constructor (message = 'Invalid parameters') { + super(message) + } +} + +export class InvalidAvgChunkSizeError extends Error { + static name = 'InvalidAvgChunkSizeError' + static code = 'ERR_INVALID_AVG_CHUNK_SIZE' + name = InvalidAvgChunkSizeError.name + code = InvalidAvgChunkSizeError.code + + constructor (message = 'Invalid avg chunk size') { + super(message) + } +} + +export class InvalidChunkSizeError extends Error { + static name = 'InvalidChunkSizeError' + static code = 'ERR_INVALID_CHUNK_SIZE' + name = InvalidChunkSizeError.name + code = InvalidChunkSizeError.code + + constructor (message = 'Invalid chunk size') { + super(message) + } +} + +export class InvalidMinChunkSizeError extends Error { + static name = 'InvalidMinChunkSizeError' + static code = 'ERR_INVALID_MIN_CHUNK_SIZE' + name = InvalidMinChunkSizeError.name + code = InvalidMinChunkSizeError.code + + constructor (message = 'Invalid min chunk size') { + super(message) + } +} + +export class InvalidContentError extends Error { + static name = 'InvalidContentError' + static code = 'ERR_INVALID_CONTENT' + name = InvalidContentError.name + code = InvalidContentError.code + + constructor (message = 'Invalid content') { + super(message) + } +} diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index aa4c3243..859b877d 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -62,13 +62,13 @@ * ``` */ -import errcode from 'err-code' import first from 'it-first' import parallelBatch from 'it-parallel-batch' import { fixedSize } from './chunker/fixed-size.js' import { type BufferImportProgressEvents, defaultBufferImporter } from './dag-builder/buffer-importer.js' import { type DAGBuilder, type DagBuilderProgressEvents, defaultDagBuilder } from './dag-builder/index.js' import { type ChunkValidator, defaultChunkValidator } from './dag-builder/validate-chunks.js' +import { InvalidParametersError } from './errors.js' import { balanced, type FileLayout } from './layout/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { Chunker } from './chunker/index.js' @@ -79,6 +79,8 @@ import type { UnixFS, Mtime } from 'ipfs-unixfs' import type { CID, Version as CIDVersion } from 'multiformats/cid' import type { ProgressOptions } from 'progress-events' +export * from './errors.js' + export type ByteStream = AwaitIterable export type ImportContent = ByteStream | Uint8Array @@ -382,7 +384,7 @@ export async function importFile (content: FileCandidate, blockstore: WritableSt const result = await first(importer([content], blockstore, options)) if (result == null) { - throw errcode(new Error('Nothing imported'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Nothing imported') } return result @@ -413,7 +415,7 @@ export async function importDirectory (content: DirectoryCandidate, blockstore: const result = await first(importer([content], blockstore, options)) if (result == null) { - throw errcode(new Error('Nothing imported'), 'ERR_INVALID_PARAMS') + throw new InvalidParametersError('Nothing imported') } return result diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index c344a864..fc28b386 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -140,7 +140,6 @@ "release": "aegir release" }, "dependencies": { - "err-code": "^3.0.1", "protons-runtime": "^5.5.0", "uint8arraylist": "^2.4.8" }, diff --git a/packages/ipfs-unixfs/src/errors.ts b/packages/ipfs-unixfs/src/errors.ts new file mode 100644 index 00000000..949caf24 --- /dev/null +++ b/packages/ipfs-unixfs/src/errors.ts @@ -0,0 +1,10 @@ +export class InvalidTypeError extends Error { + static name = 'InvalidTypeError' + static code = 'ERR_INVALID_TYPE' + name = InvalidTypeError.name + code = InvalidTypeError.code + + constructor (message = 'Invalid type') { + super(message) + } +} diff --git a/packages/ipfs-unixfs/src/index.ts b/packages/ipfs-unixfs/src/index.ts index b6263d64..8cc1ad72 100644 --- a/packages/ipfs-unixfs/src/index.ts +++ b/packages/ipfs-unixfs/src/index.ts @@ -90,7 +90,7 @@ * ``` */ -import errcode from 'err-code' +import { InvalidTypeError } from './errors.js' import { Data as PBData } from './unixfs.js' export interface Mtime { @@ -178,7 +178,7 @@ class UnixFS { } = options if (type != null && !Object.values(types).includes(type)) { - throw errcode(new Error('Type: ' + type + ' is not valid'), 'ERR_INVALID_TYPE') + throw new InvalidTypeError('Type: ' + type + ' is not valid') } this.type = type ?? 'file' @@ -250,7 +250,7 @@ class UnixFS { case 'symlink': type = PBData.DataType.Symlink; break case 'hamt-sharded-directory': type = PBData.DataType.HAMTShard; break default: - throw errcode(new Error(`Type: ${type} is not valid`), 'ERR_INVALID_TYPE') + throw new InvalidTypeError(`Type: ${type} is not valid`) } let data = this.data @@ -296,3 +296,4 @@ class UnixFS { } export { UnixFS } +export * from './errors.js' From f55cb6973433cdb83857663d651181c2654fb6ed Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 13:55:21 +0000 Subject: [PATCH 04/42] chore(release): 11.2.0 [skip ci] ## [ipfs-unixfs-v11.2.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.5...ipfs-unixfs-11.2.0) (2024-09-13) ### Features * add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 4a3848f9..1ae13450 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.5...ipfs-unixfs-11.2.0) (2024-09-13) + +### Features + +* add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) + ## [ipfs-unixfs-v11.1.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.4...ipfs-unixfs-11.1.5) (2024-09-13) ### Dependencies diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index fc28b386..91b571c4 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.1.5", + "version": "11.2.0", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From 4d5b16f0ac71362e81898a8b30e1758eb13893ca Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 13:55:37 +0000 Subject: [PATCH 05/42] chore(release): 15.3.0 [skip ci] ## [ipfs-unixfs-importer-v15.3.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.6...ipfs-unixfs-importer-15.3.0) (2024-09-13) ### Features * add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 199ca22d..10b8a1f4 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.3.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.6...ipfs-unixfs-importer-15.3.0) (2024-09-13) + +### Features + +* add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) + ## [ipfs-unixfs-importer-v15.2.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.5...ipfs-unixfs-importer-15.2.6) (2024-09-13) ### Dependencies diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 396b48df..ed2e2451 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.2.6", + "version": "15.3.0", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From 41dde042120164e6597bcead152120ebcfc6423f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 13:55:54 +0000 Subject: [PATCH 06/42] chore(release): 13.6.0 [skip ci] ## [ipfs-unixfs-exporter-v13.6.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.1...ipfs-unixfs-exporter-13.6.0) (2024-09-13) ### Features * add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index a71f6e6a..cc285ba4 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.1...ipfs-unixfs-exporter-13.6.0) (2024-09-13) + +### Features + +* add name property to errors ([#414](https://github.com/ipfs/js-ipfs-unixfs/issues/414)) ([70145c7](https://github.com/ipfs/js-ipfs-unixfs/commit/70145c7c1e78ede3c481100151a9491c639671be)) + ## [ipfs-unixfs-exporter-v13.5.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.0...ipfs-unixfs-exporter-13.5.1) (2024-09-13) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index f5faaa1e..45a0d982 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.5.1", + "version": "13.6.0", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From f0a0b37a57754ec7c36913e8ab06072aca7df41d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:01:44 +0100 Subject: [PATCH 07/42] deps(dev): bump sinon from 17.0.2 to 19.0.2 (#415) Bumps [sinon](https://github.com/sinonjs/sinon) from 17.0.2 to 19.0.2. - [Release notes](https://github.com/sinonjs/sinon/releases) - [Changelog](https://github.com/sinonjs/sinon/blob/main/docs/changelog.md) - [Commits](https://github.com/sinonjs/sinon/compare/v17.0.2...v19.0.2) --- updated-dependencies: - dependency-name: sinon dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/ipfs-unixfs-exporter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 45a0d982..1a4a312c 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -168,7 +168,7 @@ "it-to-buffer": "^4.0.7", "merge-options": "^3.0.4", "readable-stream": "^4.5.2", - "sinon": "^17.0.1", + "sinon": "^19.0.2", "uint8arrays": "^5.1.0", "wherearewe": "^2.0.1" }, From 6663007b46fb18e31157bce7bba4bfcf7b0ef5b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:01:59 +0100 Subject: [PATCH 08/42] deps(dev): bump blockstore-core from 4.4.1 to 5.0.0 (#411) Bumps [blockstore-core](https://github.com/ipfs/js-stores) from 4.4.1 to 5.0.0. - [Release notes](https://github.com/ipfs/js-stores/releases) - [Changelog](https://github.com/ipfs/js-stores/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/js-stores/compare/blockstore-core-4.4.1...blockstore-core-5.0.0) --- updated-dependencies: - dependency-name: blockstore-core dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/ipfs-unixfs-exporter/package.json | 2 +- packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 1a4a312c..4fbe9f00 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -157,7 +157,7 @@ "@types/readable-stream": "^4.0.15", "@types/sinon": "^17.0.3", "aegir": "^44.1.1", - "blockstore-core": "^4.4.1", + "blockstore-core": "^5.0.0", "delay": "^6.0.0", "ipfs-unixfs-importer": "^15.0.0", "iso-random-stream": "^2.0.2", diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index ed2e2451..53c42021 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -178,7 +178,7 @@ }, "devDependencies": { "aegir": "^44.1.1", - "blockstore-core": "^4.4.1", + "blockstore-core": "^5.0.0", "it-last": "^3.0.6", "wherearewe": "^2.0.1" }, From e3939ec99dd5e1a66a00fc249e0bd242724c43d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 16:02:08 +0100 Subject: [PATCH 09/42] deps: bump interface-store from 5.1.8 to 6.0.0 (#410) Bumps [interface-store](https://github.com/ipfs/js-stores) from 5.1.8 to 6.0.0. - [Release notes](https://github.com/ipfs/js-stores/releases) - [Changelog](https://github.com/ipfs/js-stores/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/js-stores/compare/interface-store-5.1.8...interface-store-6.0.0) --- updated-dependencies: - dependency-name: interface-store dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- packages/ipfs-unixfs-importer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 53c42021..19ecb3d5 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -164,7 +164,7 @@ "@multiformats/murmur3": "^2.1.8", "hamt-sharding": "^3.0.6", "interface-blockstore": "^5.3.0", - "interface-store": "^5.1.8", + "interface-store": "^6.0.0", "ipfs-unixfs": "^11.0.0", "it-all": "^3.0.6", "it-batch": "^3.0.6", From c30476db8de541ebe3ad88538003ee5175ec209c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 15:09:14 +0000 Subject: [PATCH 10/42] chore(release): 15.3.1 [skip ci] ## [ipfs-unixfs-importer-v15.3.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.0...ipfs-unixfs-importer-15.3.1) (2024-09-13) ### Dependencies * bump interface-store from 5.1.8 to 6.0.0 ([#410](https://github.com/ipfs/js-ipfs-unixfs/issues/410)) ([e3939ec](https://github.com/ipfs/js-ipfs-unixfs/commit/e3939ec99dd5e1a66a00fc249e0bd242724c43d0)) * **dev:** bump blockstore-core from 4.4.1 to 5.0.0 ([#411](https://github.com/ipfs/js-ipfs-unixfs/issues/411)) ([6663007](https://github.com/ipfs/js-ipfs-unixfs/commit/6663007b46fb18e31157bce7bba4bfcf7b0ef5b5)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 7 +++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 10b8a1f4..bad5711d 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,10 @@ +## [ipfs-unixfs-importer-v15.3.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.0...ipfs-unixfs-importer-15.3.1) (2024-09-13) + +### Dependencies + +* bump interface-store from 5.1.8 to 6.0.0 ([#410](https://github.com/ipfs/js-ipfs-unixfs/issues/410)) ([e3939ec](https://github.com/ipfs/js-ipfs-unixfs/commit/e3939ec99dd5e1a66a00fc249e0bd242724c43d0)) +* **dev:** bump blockstore-core from 4.4.1 to 5.0.0 ([#411](https://github.com/ipfs/js-ipfs-unixfs/issues/411)) ([6663007](https://github.com/ipfs/js-ipfs-unixfs/commit/6663007b46fb18e31157bce7bba4bfcf7b0ef5b5)) + ## [ipfs-unixfs-importer-v15.3.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.2.6...ipfs-unixfs-importer-15.3.0) (2024-09-13) ### Features diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 19ecb3d5..8216e008 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.0", + "version": "15.3.1", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From bf060cda444221225675663e2a760ef562437963 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 13 Sep 2024 15:09:33 +0000 Subject: [PATCH 11/42] chore(release): 13.6.1 [skip ci] ## [ipfs-unixfs-exporter-v13.6.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.0...ipfs-unixfs-exporter-13.6.1) (2024-09-13) ### Dependencies * **dev:** bump blockstore-core from 4.4.1 to 5.0.0 ([#411](https://github.com/ipfs/js-ipfs-unixfs/issues/411)) ([6663007](https://github.com/ipfs/js-ipfs-unixfs/commit/6663007b46fb18e31157bce7bba4bfcf7b0ef5b5)) * **dev:** bump sinon from 17.0.2 to 19.0.2 ([#415](https://github.com/ipfs/js-ipfs-unixfs/issues/415)) ([f0a0b37](https://github.com/ipfs/js-ipfs-unixfs/commit/f0a0b37a57754ec7c36913e8ab06072aca7df41d)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 7 +++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index cc285ba4..70479eea 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,10 @@ +## [ipfs-unixfs-exporter-v13.6.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.0...ipfs-unixfs-exporter-13.6.1) (2024-09-13) + +### Dependencies + +* **dev:** bump blockstore-core from 4.4.1 to 5.0.0 ([#411](https://github.com/ipfs/js-ipfs-unixfs/issues/411)) ([6663007](https://github.com/ipfs/js-ipfs-unixfs/commit/6663007b46fb18e31157bce7bba4bfcf7b0ef5b5)) +* **dev:** bump sinon from 17.0.2 to 19.0.2 ([#415](https://github.com/ipfs/js-ipfs-unixfs/issues/415)) ([f0a0b37](https://github.com/ipfs/js-ipfs-unixfs/commit/f0a0b37a57754ec7c36913e8ab06072aca7df41d)) + ## [ipfs-unixfs-exporter-v13.6.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.5.1...ipfs-unixfs-exporter-13.6.0) (2024-09-13) ### Features diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 4fbe9f00..4211c034 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.0", + "version": "13.6.1", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:50:57 +0100 Subject: [PATCH 12/42] deps(dev): bump aegir from 44.1.4 to 45.0.8 (#420) Bumps [aegir](https://github.com/ipfs/aegir) from 44.1.4 to 45.0.8. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v44.1.4...v45.0.8) --- updated-dependencies: - dependency-name: aegir dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- packages/ipfs-unixfs-exporter/package.json | 2 +- packages/ipfs-unixfs-importer/package.json | 2 +- packages/ipfs-unixfs/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index e8139b20..3ca9b93a 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "docs:no-publish": "aegir docs --publish false" }, "devDependencies": { - "aegir": "^44.1.1", + "aegir": "^45.0.8", "npm-run-all": "^4.1.5" }, "workspaces": [ diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 4211c034..e74101e4 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -156,7 +156,7 @@ "devDependencies": { "@types/readable-stream": "^4.0.15", "@types/sinon": "^17.0.3", - "aegir": "^44.1.1", + "aegir": "^45.0.8", "blockstore-core": "^5.0.0", "delay": "^6.0.0", "ipfs-unixfs-importer": "^15.0.0", diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 8216e008..6593f7e6 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -177,7 +177,7 @@ "uint8arrays": "^5.1.0" }, "devDependencies": { - "aegir": "^44.1.1", + "aegir": "^45.0.8", "blockstore-core": "^5.0.0", "it-last": "^3.0.6", "wherearewe": "^2.0.1" diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 91b571c4..26197efa 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -144,7 +144,7 @@ "uint8arraylist": "^2.4.8" }, "devDependencies": { - "aegir": "^44.1.1", + "aegir": "^45.0.8", "protons": "^7.6.0", "uint8arrays": "^5.1.0" }, From db04e6b42d794ee147b64871150be0f7a136cdc7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Mar 2025 14:56:35 +0000 Subject: [PATCH 13/42] chore(release): 11.2.1 [skip ci] ## [ipfs-unixfs-v11.2.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.0...ipfs-unixfs-11.2.1) (2025-03-06) ### Dependencies * **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 1ae13450..572e8660 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.0...ipfs-unixfs-11.2.1) (2025-03-06) + +### Dependencies + +* **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) + ## [ipfs-unixfs-v11.2.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.1.5...ipfs-unixfs-11.2.0) (2024-09-13) ### Features diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 26197efa..f445a04c 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.0", + "version": "11.2.1", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From 339afee1a00522467f9863d310f0425783ce556f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Mar 2025 14:56:51 +0000 Subject: [PATCH 14/42] chore(release): 15.3.2 [skip ci] ## [ipfs-unixfs-importer-v15.3.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.1...ipfs-unixfs-importer-15.3.2) (2025-03-06) ### Dependencies * **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index bad5711d..8033da38 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.3.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.1...ipfs-unixfs-importer-15.3.2) (2025-03-06) + +### Dependencies + +* **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) + ## [ipfs-unixfs-importer-v15.3.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.0...ipfs-unixfs-importer-15.3.1) (2024-09-13) ### Dependencies diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 6593f7e6..63abf473 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.1", + "version": "15.3.2", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From 33894da536f736b73dfc62c94bffffc3f42360ae Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 6 Mar 2025 14:57:09 +0000 Subject: [PATCH 15/42] chore(release): 13.6.2 [skip ci] ## [ipfs-unixfs-exporter-v13.6.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.1...ipfs-unixfs-exporter-13.6.2) (2025-03-06) ### Dependencies * **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 70479eea..4ba0889c 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.1...ipfs-unixfs-exporter-13.6.2) (2025-03-06) + +### Dependencies + +* **dev:** bump aegir from 44.1.4 to 45.0.8 ([#420](https://github.com/ipfs/js-ipfs-unixfs/issues/420)) ([6eb1064](https://github.com/ipfs/js-ipfs-unixfs/commit/6eb1064ceaf3bbbdadc639e9641f3d9fad8ab23b)) + ## [ipfs-unixfs-exporter-v13.6.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.0...ipfs-unixfs-exporter-13.6.1) (2024-09-13) ### Dependencies diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index e74101e4..b81626c1 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.1", + "version": "13.6.2", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 7b178e98a40661059780dc16b6c630a261e1c38e Mon Sep 17 00:00:00 2001 From: web3-bot Date: Fri, 28 Mar 2025 14:08:08 +0100 Subject: [PATCH 16/42] ci: uci/copy-templates (#423) * chore: add or force update .github/workflows/stale.yml * chore: add or force update .github/workflows/generated-pr.yml --- .github/workflows/generated-pr.yml | 14 ++++++++++++++ .github/workflows/stale.yml | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/generated-pr.yml diff --git a/.github/workflows/generated-pr.yml b/.github/workflows/generated-pr.yml new file mode 100644 index 00000000..b8c5cc63 --- /dev/null +++ b/.github/workflows/generated-pr.yml @@ -0,0 +1,14 @@ +name: Close Generated PRs + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 16d65d72..7c955c41 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,8 +1,9 @@ -name: Close and mark stale issue +name: Close Stale Issues on: schedule: - cron: '0 0 * * *' + workflow_dispatch: permissions: issues: write @@ -10,4 +11,4 @@ permissions: jobs: stale: - uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3 + uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 From 8ca014420094be90b8bb765bb3f703a9ce7260b1 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 16 Jun 2025 15:39:30 +0300 Subject: [PATCH 17/42] fix: limit incoming hamt width (#433) Matches the undocumented limit in boxo. --- packages/ipfs-unixfs/src/errors.ts | 11 +++++++++++ packages/ipfs-unixfs/src/index.ts | 9 ++++++++- packages/ipfs-unixfs/test/unixfs-format.spec.ts | 13 +++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/src/errors.ts b/packages/ipfs-unixfs/src/errors.ts index 949caf24..6ac59fed 100644 --- a/packages/ipfs-unixfs/src/errors.ts +++ b/packages/ipfs-unixfs/src/errors.ts @@ -8,3 +8,14 @@ export class InvalidTypeError extends Error { super(message) } } + +export class InvalidUnixFSMessageError extends Error { + static name = 'InvalidUnixFSMessageError' + static code = 'ERR_INVALID_MESSAGE' + name = InvalidUnixFSMessageError.name + code = InvalidUnixFSMessageError.code + + constructor (message = 'Invalid message') { + super(message) + } +} diff --git a/packages/ipfs-unixfs/src/index.ts b/packages/ipfs-unixfs/src/index.ts index 8cc1ad72..82f76823 100644 --- a/packages/ipfs-unixfs/src/index.ts +++ b/packages/ipfs-unixfs/src/index.ts @@ -90,7 +90,7 @@ * ``` */ -import { InvalidTypeError } from './errors.js' +import { InvalidTypeError, InvalidUnixFSMessageError } from './errors.js' import { Data as PBData } from './unixfs.js' export interface Mtime { @@ -117,6 +117,9 @@ const dirTypes = [ const DEFAULT_FILE_MODE = parseInt('0644', 8) const DEFAULT_DIRECTORY_MODE = parseInt('0755', 8) +// https://github.com/ipfs/boxo/blob/364c5040ec91ec8e2a61446e9921e9225704c34d/ipld/unixfs/hamt/hamt.go#L778 +const MAX_FANOUT = BigInt(1 << 10) + export interface UnixFSOptions { type?: string data?: Uint8Array @@ -134,6 +137,10 @@ class UnixFS { static unmarshal (marshaled: Uint8Array): UnixFS { const message = PBData.decode(marshaled) + if (message.fanout != null && message.fanout > MAX_FANOUT) { + throw new InvalidUnixFSMessageError(`Fanout size was too large - ${message.fanout} > ${MAX_FANOUT}`) + } + const data = new UnixFS({ type: types[message.Type != null ? message.Type.toString() : 'File'], data: message.Data, diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.ts b/packages/ipfs-unixfs/test/unixfs-format.spec.ts index a15d0f2e..12dae55b 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.ts +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.ts @@ -431,4 +431,17 @@ describe('unixfs-format', () => { expect(marshaled).to.deep.equal(Uint8Array.from([0x08, 0x02, 0x18, 0x00])) }) + + it('should limit maximum fanout size', () => { + const data = new UnixFS({ + type: 'hamt-sharded-directory', + fanout: 1025n + }) + const marshaled = data.marshal() + + expect(() => { + UnixFS.unmarshal(marshaled) + }).to.throw() + .with.property('name', 'InvalidUnixFSMessageError') + }) }) From 89006d7e907be06477ac77fd678036f9ce215219 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Jun 2025 12:45:17 +0000 Subject: [PATCH 18/42] chore(release): 11.2.2 [skip ci] ## [ipfs-unixfs-v11.2.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.1...ipfs-unixfs-11.2.2) (2025-06-16) ### Bug Fixes * limit incoming hamt width ([#433](https://github.com/ipfs/js-ipfs-unixfs/issues/433)) ([8ca0144](https://github.com/ipfs/js-ipfs-unixfs/commit/8ca014420094be90b8bb765bb3f703a9ce7260b1)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 572e8660..956b1de1 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.1...ipfs-unixfs-11.2.2) (2025-06-16) + +### Bug Fixes + +* limit incoming hamt width ([#433](https://github.com/ipfs/js-ipfs-unixfs/issues/433)) ([8ca0144](https://github.com/ipfs/js-ipfs-unixfs/commit/8ca014420094be90b8bb765bb3f703a9ce7260b1)) + ## [ipfs-unixfs-v11.2.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.0...ipfs-unixfs-11.2.1) (2025-03-06) ### Dependencies diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index f445a04c..ef24b7c4 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.1", + "version": "11.2.2", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From 14d7e1cacfa61fad6df86c0875ad52ce6b203457 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 16:20:47 +0300 Subject: [PATCH 19/42] deps(dev): bump sinon from 19.0.5 to 21.0.0 (#432) Bumps [sinon](https://github.com/sinonjs/sinon) from 19.0.5 to 21.0.0. - [Release notes](https://github.com/sinonjs/sinon/releases) - [Changelog](https://github.com/sinonjs/sinon/blob/main/docs/changelog.md) - [Commits](https://github.com/sinonjs/sinon/commits) --- updated-dependencies: - dependency-name: sinon dependency-version: 21.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/ipfs-unixfs-exporter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index b81626c1..8e516a1c 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -168,7 +168,7 @@ "it-to-buffer": "^4.0.7", "merge-options": "^3.0.4", "readable-stream": "^4.5.2", - "sinon": "^19.0.2", + "sinon": "^21.0.0", "uint8arrays": "^5.1.0", "wherearewe": "^2.0.1" }, From e059bbdf2aed84198407c29746c3a8ce46bd3fd4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 16 Jun 2025 13:26:36 +0000 Subject: [PATCH 20/42] chore(release): 13.6.3 [skip ci] ## [ipfs-unixfs-exporter-v13.6.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.2...ipfs-unixfs-exporter-13.6.3) (2025-06-16) ### Dependencies * **dev:** bump sinon from 19.0.5 to 21.0.0 ([#432](https://github.com/ipfs/js-ipfs-unixfs/issues/432)) ([14d7e1c](https://github.com/ipfs/js-ipfs-unixfs/commit/14d7e1cacfa61fad6df86c0875ad52ce6b203457)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 4ba0889c..3d046c17 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.2...ipfs-unixfs-exporter-13.6.3) (2025-06-16) + +### Dependencies + +* **dev:** bump sinon from 19.0.5 to 21.0.0 ([#432](https://github.com/ipfs/js-ipfs-unixfs/issues/432)) ([14d7e1c](https://github.com/ipfs/js-ipfs-unixfs/commit/14d7e1cacfa61fad6df86c0875ad52ce6b203457)) + ## [ipfs-unixfs-exporter-v13.6.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.1...ipfs-unixfs-exporter-13.6.2) (2025-03-06) ### Dependencies diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 8e516a1c..3c2141d3 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.2", + "version": "13.6.3", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 1fb2db37f33674eb7a0e00aa88d5312a7644536d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 07:25:08 +0300 Subject: [PATCH 21/42] deps: bump aegir from 45.2.1 to 47.0.16 (#431) Bumps [aegir](https://github.com/ipfs/aegir) from 45.2.1 to 47.0.16. - [Release notes](https://github.com/ipfs/aegir/releases) - [Changelog](https://github.com/ipfs/aegir/blob/main/CHANGELOG.md) - [Commits](https://github.com/ipfs/aegir/compare/v45.2.1...v47.0.16) --- updated-dependencies: - dependency-name: aegir dependency-version: 47.0.16 dependency-type: direct:development update-type: version-update:semver-major ... --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alex Potsides --- .github/dependabot.yml | 23 +- .github/workflows/semantic-pull-request.yml | 2 +- .github/workflows/stale.yml | 2 +- .gitignore | 2 + README.md | 6 +- benchmarks/import/package.json | 38 +--- benchmarks/import/test/index.spec.ts | 5 +- benchmarks/import/tsconfig.json | 5 + benchmarks/memory/package.json | 31 +-- benchmarks/memory/test/index.spec.ts | 10 +- benchmarks/memory/tsconfig.json | 5 + package.json | 3 +- .../ipfs-unixfs-exporter/CODE_OF_CONDUCT.md | 3 + packages/ipfs-unixfs-exporter/LICENSE | 4 - packages/ipfs-unixfs-exporter/LICENSE-APACHE | 202 +++++++++++++++++- packages/ipfs-unixfs-exporter/package.json | 57 ++--- .../src/resolvers/unixfs-v1/content/file.ts | 3 +- .../content/hamt-sharded-directory.ts | 3 +- .../src/resolvers/unixfs-v1/index.ts | 3 +- .../src/utils/find-cid-in-shard.ts | 6 +- .../test/exporter-sharded.spec.ts | 3 +- .../test/exporter.spec.ts | 3 +- .../test/import-export-dir-sharding.spec.ts | 3 +- .../test/import-export-nested-dir.spec.ts | 7 +- .../test/import-export.spec.ts | 6 +- .../test/importer.spec.ts | 18 +- packages/ipfs-unixfs-exporter/typedoc.json | 1 + .../ipfs-unixfs-importer/CODE_OF_CONDUCT.md | 3 + packages/ipfs-unixfs-importer/LICENSE | 4 - packages/ipfs-unixfs-importer/LICENSE-APACHE | 202 +++++++++++++++++- packages/ipfs-unixfs-importer/package.json | 39 ++-- .../src/dag-builder/buffer-importer.ts | 3 +- .../src/dag-builder/file.ts | 3 +- .../src/dag-builder/index.ts | 6 +- packages/ipfs-unixfs-importer/src/dir-flat.ts | 13 +- .../ipfs-unixfs-importer/src/dir-sharded.ts | 16 +- packages/ipfs-unixfs-importer/src/dir.ts | 2 +- .../ipfs-unixfs-importer/src/flat-to-shard.ts | 5 +- packages/ipfs-unixfs-importer/src/index.ts | 12 +- .../ipfs-unixfs-importer/src/tree-builder.ts | 2 +- .../test/hash-parity-with-go-ipfs.spec.ts | 6 +- packages/ipfs-unixfs-importer/typedoc.json | 1 + packages/ipfs-unixfs/CODE_OF_CONDUCT.md | 3 + packages/ipfs-unixfs/LICENSE | 4 - packages/ipfs-unixfs/LICENSE-APACHE | 202 +++++++++++++++++- packages/ipfs-unixfs/package.json | 24 +-- packages/ipfs-unixfs/src/unixfs.ts | 6 - .../ipfs-unixfs/test/unixfs-format.spec.ts | 3 +- packages/ipfs-unixfs/typedoc.json | 1 + typedoc.json | 9 +- 50 files changed, 820 insertions(+), 203 deletions(-) create mode 100644 packages/ipfs-unixfs-exporter/CODE_OF_CONDUCT.md delete mode 100644 packages/ipfs-unixfs-exporter/LICENSE create mode 100644 packages/ipfs-unixfs-importer/CODE_OF_CONDUCT.md delete mode 100644 packages/ipfs-unixfs-importer/LICENSE create mode 100644 packages/ipfs-unixfs/CODE_OF_CONDUCT.md delete mode 100644 packages/ipfs-unixfs/LICENSE diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d401a774..589a3341 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,30 @@ version: 2 updates: - package-ecosystem: npm - directory: "/" + directories: + - "/" schedule: interval: daily time: "10:00" open-pull-requests-limit: 20 commit-message: prefix: "deps" - prefix-development: "deps(dev)" + prefix-development: "chore" + groups: + interplanetary-deps: # Helia/libp2p deps + patterns: + - "*helia*" + - "*libp2p*" + - "*multiformats*" + - "*blockstore*" + - "*datastore*" + kubo-deps: # kubo deps + patterns: + - "*kubo*" + - "ipfsd-ctl" +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: chore diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml index bd00f090..8c0b9eaf 100644 --- a/.github/workflows/semantic-pull-request.yml +++ b/.github/workflows/semantic-pull-request.yml @@ -9,4 +9,4 @@ on: jobs: main: - uses: pl-strflt/.github/.github/workflows/reusable-semantic-pull-request.yml@v0.3 + uses: ipdxco/unified-github-workflows/.github/workflows/reusable-semantic-pull-request.yml@v1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7c955c41..1f9d6b8c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: Close Stale Issues +name: Close and mark stale issue on: schedule: diff --git a/.gitignore b/.gitignore index 7ad9e674..f115d26b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ node_modules package-lock.json yarn.lock .vscode +.tmp-compiled-docs +tsconfig-doc-check.aegir.json diff --git a/README.md b/README.md index 17c2080a..9ff2f886 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,16 @@ > JS implementation of the IPFS UnixFS +The UnixFS spec can be found at [ipfs/specs/UNIXFS.md](https://github.com/ipfs/specs/blob/master/UNIXFS.md) + # Packages +- [`benchmarks/import`](https://github.com/ipfs/js-ipfs-unixfs/tree/main/benchmarks/import) Import benchmarks for ipfs-unixfs-importer +- [`benchmarks/memory`](https://github.com/ipfs/js-ipfs-unixfs/tree/main/benchmarks/memory) Memory benchmarks for ipfs-unixfs-importer - [`packages/ipfs-unixfs`](https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs) JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG) - [`packages/ipfs-unixfs-exporter`](https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter) JavaScript implementation of the UnixFs exporter used by IPFS - [`packages/ipfs-unixfs-importer`](https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer) JavaScript implementation of the UnixFs importer used by IPFS -The UnixFS spec can be found at [ipfs/specs/UNIXFS.md](https://github.com/ipfs/specs/blob/master/UNIXFS.md) - # API Docs - diff --git a/benchmarks/import/package.json b/benchmarks/import/package.json index fa9f06ab..2c380ca6 100644 --- a/benchmarks/import/package.json +++ b/benchmarks/import/package.json @@ -1,29 +1,9 @@ { - "name": "ipfs-unixfs-memory-benchmark", + "name": "ipfs-unixfs-import-benchmark", "version": "0.0.0", - "description": "Memory benchmarks for ipfs-unixfs-importer", + "description": "Import benchmarks for ipfs-unixfs-importer", "license": "Apache-2.0 OR MIT", - "private": true, "type": "module", - "types": "./dist/src/index.d.ts", - "files": [ - "src", - "dist", - "!dist/test", - "!**/*.tsbuildinfo" - ], - "exports": { - ".": { - "types": "./dist/src/index.d.ts", - "import": "./dist/src/index.js" - } - }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "sourceType": "module" - } - }, "scripts": { "build": "aegir build --bundle false", "clean": "aegir clean", @@ -32,11 +12,11 @@ "start": "npm run build && node --expose-gc ./dist/test/index.spec.js" }, "devDependencies": { - "aegir": "^42.2.2", - "blockstore-core": "^4.0.1", - "blockstore-fs": "^1.0.0", - "ipfs-unixfs-importer": "../../packages/ipfs-unixfs-importer", - "it-buffer-stream": "^3.0.1", - "it-drain": "^3.0.5" - } + "aegir": "^47.0.16", + "blockstore-core": "^5.0.4", + "ipfs-unixfs-importer": "^15.0.0", + "it-buffer-stream": "^3.0.11", + "it-drain": "^3.0.10" + }, + "private": true } diff --git a/benchmarks/import/test/index.spec.ts b/benchmarks/import/test/index.spec.ts index dbd65ced..cb7a2df1 100644 --- a/benchmarks/import/test/index.spec.ts +++ b/benchmarks/import/test/index.spec.ts @@ -1,9 +1,10 @@ /* eslint-env mocha */ -import { importer, ImporterOptions } from 'ipfs-unixfs-importer' -import bufferStream from 'it-buffer-stream' import { MemoryBlockstore } from 'blockstore-core' +import { importer } from 'ipfs-unixfs-importer' +import bufferStream from 'it-buffer-stream' import drain from 'it-drain' +import type { ImporterOptions } from 'ipfs-unixfs-importer' const REPEATS = 10 const FILE_SIZE = Math.pow(2, 20) * 500 // 500MB diff --git a/benchmarks/import/tsconfig.json b/benchmarks/import/tsconfig.json index 13a35996..304093f2 100644 --- a/benchmarks/import/tsconfig.json +++ b/benchmarks/import/tsconfig.json @@ -6,5 +6,10 @@ "include": [ "src", "test" + ], + "references": [ + { + "path": "../../packages/ipfs-unixfs-importer" + } ] } diff --git a/benchmarks/memory/package.json b/benchmarks/memory/package.json index ae510405..70f46164 100644 --- a/benchmarks/memory/package.json +++ b/benchmarks/memory/package.json @@ -3,27 +3,7 @@ "version": "0.0.0", "description": "Memory benchmarks for ipfs-unixfs-importer", "license": "Apache-2.0 OR MIT", - "private": true, "type": "module", - "types": "./dist/src/index.d.ts", - "files": [ - "src", - "dist", - "!dist/test", - "!**/*.tsbuildinfo" - ], - "exports": { - ".": { - "types": "./dist/src/index.d.ts", - "import": "./dist/src/index.js" - } - }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "sourceType": "module" - } - }, "scripts": { "build": "aegir build --bundle false", "clean": "aegir clean", @@ -32,9 +12,10 @@ "start": "npm run build && node --expose-gc ./dist/test/index.spec.js" }, "devDependencies": { - "aegir": "^42.2.2", - "blockstore-fs": "^1.0.0", - "ipfs-unixfs-importer": "../../packages/ipfs-unixfs-importer", - "it-drain": "^3.0.5" - } + "aegir": "^47.0.16", + "blockstore-fs": "^2.0.4", + "ipfs-unixfs-importer": "^15.0.0", + "it-drain": "^3.0.10" + }, + "private": true } diff --git a/benchmarks/memory/test/index.spec.ts b/benchmarks/memory/test/index.spec.ts index 6a02873a..a42603da 100644 --- a/benchmarks/memory/test/index.spec.ts +++ b/benchmarks/memory/test/index.spec.ts @@ -1,11 +1,11 @@ /* eslint-disable no-console */ -import { importer } from 'ipfs-unixfs-importer' -import path from 'node:path' -import os from 'node:os' import fs from 'node:fs' -import drain from 'it-drain' +import os from 'node:os' +import path from 'node:path' import { FsBlockstore } from 'blockstore-fs' +import { importer } from 'ipfs-unixfs-importer' +import drain from 'it-drain' const ONE_MEG = 1024 * 1024 @@ -43,6 +43,6 @@ async function main (): Promise { } main().catch(err => { - console.error(err) // eslint-disable-line no-console + console.error(err) process.exit(1) }) diff --git a/benchmarks/memory/tsconfig.json b/benchmarks/memory/tsconfig.json index 13a35996..304093f2 100644 --- a/benchmarks/memory/tsconfig.json +++ b/benchmarks/memory/tsconfig.json @@ -6,5 +6,10 @@ "include": [ "src", "test" + ], + "references": [ + { + "path": "../../packages/ipfs-unixfs-importer" + } ] } diff --git a/package.json b/package.json index 3ca9b93a..13c17a6f 100644 --- a/package.json +++ b/package.json @@ -33,10 +33,11 @@ "docs:no-publish": "aegir docs --publish false" }, "devDependencies": { - "aegir": "^45.0.8", + "aegir": "^47.0.16", "npm-run-all": "^4.1.5" }, "workspaces": [ + "benchmarks/*", "packages/*" ] } diff --git a/packages/ipfs-unixfs-exporter/CODE_OF_CONDUCT.md b/packages/ipfs-unixfs-exporter/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..6b0fa54c --- /dev/null +++ b/packages/ipfs-unixfs-exporter/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Contributor Code of Conduct + +This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) diff --git a/packages/ipfs-unixfs-exporter/LICENSE b/packages/ipfs-unixfs-exporter/LICENSE deleted file mode 100644 index 20ce483c..00000000 --- a/packages/ipfs-unixfs-exporter/LICENSE +++ /dev/null @@ -1,4 +0,0 @@ -This project is dual licensed under MIT and Apache-2.0. - -MIT: https://www.opensource.org/licenses/mit -Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/ipfs-unixfs-exporter/LICENSE-APACHE b/packages/ipfs-unixfs-exporter/LICENSE-APACHE index 14478a3b..b09cd785 100644 --- a/packages/ipfs-unixfs-exporter/LICENSE-APACHE +++ b/packages/ipfs-unixfs-exporter/LICENSE-APACHE @@ -1,5 +1,201 @@ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 3c2141d3..f77b4eab 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -32,13 +32,6 @@ "import": "./dist/src/index.js" } }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "project": true, - "sourceType": "module" - } - }, "release": { "branches": [ "main" @@ -121,7 +114,15 @@ "@semantic-release/changelog", "@semantic-release/npm", "@semantic-release/github", - "@semantic-release/git" + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md", + "package.json" + ] + } + ] ] }, "scripts": { @@ -136,38 +137,38 @@ "release": "aegir release" }, "dependencies": { - "@ipld/dag-cbor": "^9.2.1", - "@ipld/dag-json": "^10.2.2", - "@ipld/dag-pb": "^4.1.2", + "@ipld/dag-cbor": "^9.2.4", + "@ipld/dag-json": "^10.2.5", + "@ipld/dag-pb": "^4.1.5", "@multiformats/murmur3": "^2.1.8", "hamt-sharding": "^3.0.6", - "interface-blockstore": "^5.3.0", + "interface-blockstore": "^5.3.2", "ipfs-unixfs": "^11.0.0", - "it-filter": "^3.1.1", - "it-last": "^3.0.6", - "it-map": "^3.1.1", - "it-parallel": "^3.0.8", + "it-filter": "^3.1.4", + "it-last": "^3.0.9", + "it-map": "^3.1.4", + "it-parallel": "^3.0.13", "it-pipe": "^3.0.1", "it-pushable": "^3.2.3", - "multiformats": "^13.2.3", - "p-queue": "^8.0.1", + "multiformats": "^13.3.7", + "p-queue": "^8.1.0", "progress-events": "^1.0.1" }, "devDependencies": { - "@types/readable-stream": "^4.0.15", - "@types/sinon": "^17.0.3", - "aegir": "^45.0.8", - "blockstore-core": "^5.0.0", + "@types/readable-stream": "^4.0.21", + "@types/sinon": "^17.0.4", + "aegir": "^47.0.16", + "blockstore-core": "^5.0.4", "delay": "^6.0.0", "ipfs-unixfs-importer": "^15.0.0", "iso-random-stream": "^2.0.2", - "it-all": "^3.0.6", - "it-buffer-stream": "^3.0.8", - "it-drain": "^3.0.7", - "it-first": "^3.0.6", - "it-to-buffer": "^4.0.7", + "it-all": "^3.0.9", + "it-buffer-stream": "^3.0.11", + "it-drain": "^3.0.10", + "it-first": "^3.0.9", + "it-to-buffer": "^4.0.10", "merge-options": "^3.0.4", - "readable-stream": "^4.5.2", + "readable-stream": "^4.7.0", "sinon": "^21.0.0", "uint8arrays": "^5.1.0", "wherearewe": "^2.0.1" diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts index d88e1a24..3d549048 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.ts @@ -3,7 +3,7 @@ import { UnixFS } from 'ipfs-unixfs' import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' -import { type Pushable, pushable } from 'it-pushable' +import { pushable } from 'it-pushable' import * as raw from 'multiformats/codecs/raw' import PQueue from 'p-queue' import { CustomProgressEvent } from 'progress-events' @@ -11,6 +11,7 @@ import { NotUnixFSError, OverReadError, UnderReadError } from '../../../errors.j import extractDataFromBlock from '../../../utils/extract-data-from-block.js' import validateOffsetAndLength from '../../../utils/validate-offset-and-length.js' import type { ExporterOptions, UnixfsV1FileContent, UnixfsV1Resolver, ReadableStorage, ExportProgress, ExportWalk } from '../../../index.js' +import type { Pushable } from 'it-pushable' async function walkDAG (blockstore: ReadableStorage, node: dagPb.PBNode | Uint8Array, queue: Pushable, streamPosition: bigint, start: bigint, end: bigint, options: ExporterOptions): Promise { // a `raw` node diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index 52c2938d..b08255a2 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -1,4 +1,4 @@ -import { decode, type PBNode } from '@ipld/dag-pb' +import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import map from 'it-map' import parallel from 'it-parallel' @@ -6,6 +6,7 @@ import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' +import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 2ed6c6a0..4c5983e4 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -1,4 +1,4 @@ -import { decode, type PBNode } from '@ipld/dag-pb' +import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' @@ -6,6 +6,7 @@ import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' const findLinkCid = (node: PBNode, name: string): CID | undefined => { diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts index d3446ddd..d604ff28 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.ts @@ -1,9 +1,11 @@ -import { decode, type PBLink, type PBNode } from '@ipld/dag-pb' +import { decode } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' -import { Bucket, type BucketPosition, createHAMT } from 'hamt-sharding' +import { Bucket, createHAMT } from 'hamt-sharding' import { UnixFS } from 'ipfs-unixfs' import { NotUnixFSError } from '../errors.js' import type { ExporterOptions, ShardTraversalContext, ReadableStorage } from '../index.js' +import type { PBLink, PBNode } from '@ipld/dag-pb' +import type { BucketPosition } from 'hamt-sharding' import type { CID } from 'multiformats/cid' // FIXME: this is copy/pasted from ipfs-unixfs-importer/src/options.js diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index 00a3f1a8..dd2354c5 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -4,7 +4,7 @@ import * as dagPb from '@ipld/dag-pb' import { expect } from 'aegir/chai' import { MemoryBlockstore } from 'blockstore-core' import { UnixFS } from 'ipfs-unixfs' -import { importer, type ImportCandidate } from 'ipfs-unixfs-importer' +import { importer } from 'ipfs-unixfs-importer' import all from 'it-all' import randomBytes from 'it-buffer-stream' import last from 'it-last' @@ -13,6 +13,7 @@ import { sha256 } from 'multiformats/hashes/sha2' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { exporter, walkPath } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' +import type { ImportCandidate } from 'ipfs-unixfs-importer' const SHARD_SPLIT_THRESHOLD = 10 diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index c1bce7d9..a48fe9bb 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -9,7 +9,7 @@ import delay from 'delay' import { UnixFS } from 'ipfs-unixfs' import { importer } from 'ipfs-unixfs-importer' import { fixedSize } from 'ipfs-unixfs-importer/chunker' -import { balanced, type FileLayout, flat, trickle } from 'ipfs-unixfs-importer/layout' +import { balanced, flat, trickle } from 'ipfs-unixfs-importer/layout' import all from 'it-all' import randomBytes from 'it-buffer-stream' import drain from 'it-drain' @@ -32,6 +32,7 @@ import asAsyncIterable from './helpers/as-async-iterable.js' import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' import type { Chunker } from 'ipfs-unixfs-importer/chunker' +import type { FileLayout } from 'ipfs-unixfs-importer/layout' const ONE_MEG = Math.pow(1024, 2) diff --git a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.ts b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.ts index 2f4d5451..7140ea52 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.ts @@ -8,8 +8,9 @@ import last from 'it-last' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { exporter, type UnixFSDirectory, type UnixFSEntry } from '../src/index.js' +import { exporter } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' +import type { UnixFSDirectory, UnixFSEntry } from '../src/index.js' import type { CID } from 'multiformats/cid' describe('builder: directory sharding', () => { diff --git a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.ts b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.ts index e1ef084a..46dd856c 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.ts @@ -7,8 +7,9 @@ import all from 'it-all' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { exporter, type UnixFSEntry } from '../src/index.js' +import { exporter } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' +import type { UnixFSEntry } from '../src/index.js' import type { CID } from 'multiformats/cid' describe('import and export: directory', () => { @@ -113,7 +114,7 @@ function normalizeNode (node: { path?: string, cid: CID }): { path: string, mult } function byPath (a: { path: string }, b: { path: string }): number { - if (a.path > b.path) return -1 - if (a.path < b.path) return 1 + if (a.path > b.path) { return -1 } + if (a.path < b.path) { return 1 } return 0 } diff --git a/packages/ipfs-unixfs-exporter/test/import-export.spec.ts b/packages/ipfs-unixfs-exporter/test/import-export.spec.ts index de7e6479..82b8c24e 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/import-export.spec.ts @@ -4,10 +4,12 @@ import { expect } from 'aegir/chai' import loadFixture from 'aegir/fixtures' import { MemoryBlockstore } from 'blockstore-core' -import { importer, type ImporterOptions } from 'ipfs-unixfs-importer' -import { flat, balanced, trickle, type FileLayout } from 'ipfs-unixfs-importer/layout' +import { importer } from 'ipfs-unixfs-importer' +import { flat, balanced, trickle } from 'ipfs-unixfs-importer/layout' import { exporter } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' +import type { ImporterOptions } from 'ipfs-unixfs-importer' +import type { FileLayout } from 'ipfs-unixfs-importer/layout' const bigFile = loadFixture(('test') + '/fixtures/1.2MiB.txt') diff --git a/packages/ipfs-unixfs-exporter/test/importer.spec.ts b/packages/ipfs-unixfs-exporter/test/importer.spec.ts index 8a354159..2684468b 100644 --- a/packages/ipfs-unixfs-exporter/test/importer.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/importer.spec.ts @@ -4,13 +4,14 @@ import { decode } from '@ipld/dag-pb' import { expect } from 'aegir/chai' import loadFixture from 'aegir/fixtures' import { MemoryBlockstore } from 'blockstore-core' -import { type Mtime, UnixFS } from 'ipfs-unixfs' -import { importer, type ImporterOptions } from 'ipfs-unixfs-importer' +import { UnixFS } from 'ipfs-unixfs' +import { importer } from 'ipfs-unixfs-importer' import { fixedSize } from 'ipfs-unixfs-importer/chunker' -import { balanced, type FileLayout, flat, trickle } from 'ipfs-unixfs-importer/layout' +import { balanced, flat, trickle } from 'ipfs-unixfs-importer/layout' import all from 'it-all' import first from 'it-first' import last from 'it-last' +// @ts-expect-error https://github.com/schnittstabil/merge-options/pull/28 import extend from 'merge-options' import { base58btc } from 'multiformats/bases/base58' import { CID } from 'multiformats/cid' @@ -21,6 +22,9 @@ import { exporter, recursive } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' import collectLeafCids from './helpers/collect-leaf-cids.js' import type { Blockstore } from 'interface-blockstore' +import type { Mtime } from 'ipfs-unixfs' +import type { ImporterOptions } from 'ipfs-unixfs-importer' +import type { FileLayout } from 'ipfs-unixfs-importer/layout' const bigFile = loadFixture('test/fixtures/1.2MiB.txt') const smallFile = loadFixture('test/fixtures/200Bytes.txt') @@ -1074,7 +1078,7 @@ describe('configuration', () => { content: 'content' }], block, { /** @type {import('ipfs-unixfs-importer').DAGBuilder} */ - dagBuilder: async function * (source, block) { // eslint-disable-line require-await + dagBuilder: async function * (source, block) { yield async function () { return Promise.resolve({ cid, @@ -1085,7 +1089,7 @@ describe('configuration', () => { } }, /** @type {import('ipfs-unixfs-importer').TreeBuilder} */ - treeBuilder: async function * (source, block) { // eslint-disable-line require-await + treeBuilder: async function * (source, block) { builtTree = true yield * source } @@ -1106,7 +1110,7 @@ describe('configuration', () => { path: 'path', content: asAsyncIterable(uint8ArrayFromString('content')) }], block, { - chunkValidator: async function * (source) { // eslint-disable-line require-await + chunkValidator: async function * (source) { validated = true for await (const str of source) { @@ -1117,7 +1121,7 @@ describe('configuration', () => { } } }, - chunker: async function * (source) { // eslint-disable-line require-await + chunker: async function * (source) { chunked = true yield * source } diff --git a/packages/ipfs-unixfs-exporter/typedoc.json b/packages/ipfs-unixfs-exporter/typedoc.json index f599dc72..db0b0747 100644 --- a/packages/ipfs-unixfs-exporter/typedoc.json +++ b/packages/ipfs-unixfs-exporter/typedoc.json @@ -1,4 +1,5 @@ { + "readme": "none", "entryPoints": [ "./src/index.ts" ] diff --git a/packages/ipfs-unixfs-importer/CODE_OF_CONDUCT.md b/packages/ipfs-unixfs-importer/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..6b0fa54c --- /dev/null +++ b/packages/ipfs-unixfs-importer/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Contributor Code of Conduct + +This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) diff --git a/packages/ipfs-unixfs-importer/LICENSE b/packages/ipfs-unixfs-importer/LICENSE deleted file mode 100644 index 20ce483c..00000000 --- a/packages/ipfs-unixfs-importer/LICENSE +++ /dev/null @@ -1,4 +0,0 @@ -This project is dual licensed under MIT and Apache-2.0. - -MIT: https://www.opensource.org/licenses/mit -Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/ipfs-unixfs-importer/LICENSE-APACHE b/packages/ipfs-unixfs-importer/LICENSE-APACHE index 14478a3b..b09cd785 100644 --- a/packages/ipfs-unixfs-importer/LICENSE-APACHE +++ b/packages/ipfs-unixfs-importer/LICENSE-APACHE @@ -1,5 +1,201 @@ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 63abf473..0511a28e 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -56,13 +56,6 @@ "import": "./dist/src/layout/index.js" } }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "project": true, - "sourceType": "module" - } - }, "release": { "branches": [ "main" @@ -145,7 +138,15 @@ "@semantic-release/changelog", "@semantic-release/npm", "@semantic-release/github", - "@semantic-release/git" + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md", + "package.json" + ] + } + ] ] }, "scripts": { @@ -160,26 +161,26 @@ "release": "aegir release" }, "dependencies": { - "@ipld/dag-pb": "^4.1.2", + "@ipld/dag-pb": "^4.1.5", "@multiformats/murmur3": "^2.1.8", "hamt-sharding": "^3.0.6", - "interface-blockstore": "^5.3.0", - "interface-store": "^6.0.0", + "interface-blockstore": "^5.3.2", + "interface-store": "^6.0.3", "ipfs-unixfs": "^11.0.0", - "it-all": "^3.0.6", - "it-batch": "^3.0.6", - "it-first": "^3.0.6", - "it-parallel-batch": "^3.0.6", - "multiformats": "^13.2.3", + "it-all": "^3.0.9", + "it-batch": "^3.0.9", + "it-first": "^3.0.9", + "it-parallel-batch": "^3.0.9", + "multiformats": "^13.3.7", "progress-events": "^1.0.1", "rabin-wasm": "^0.1.5", "uint8arraylist": "^2.4.8", "uint8arrays": "^5.1.0" }, "devDependencies": { - "aegir": "^45.0.8", - "blockstore-core": "^5.0.0", - "it-last": "^3.0.6", + "aegir": "^47.0.16", + "blockstore-core": "^5.0.4", + "it-last": "^3.0.9", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/buffer-importer.ts b/packages/ipfs-unixfs-importer/src/dag-builder/buffer-importer.ts index c52b7285..b4f033ae 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/buffer-importer.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/buffer-importer.ts @@ -2,8 +2,9 @@ import * as dagPb from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import * as raw from 'multiformats/codecs/raw' import { CustomProgressEvent } from 'progress-events' -import { persist, type PersistOptions } from '../utils/persist.js' +import { persist } from '../utils/persist.js' import type { BufferImporter } from '../index.js' +import type { PersistOptions } from '../utils/persist.js' import type { CID, Version } from 'multiformats/cid' import type { ProgressOptions, ProgressEvent } from 'progress-events' diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 3c3b19af..41862bce 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -1,4 +1,4 @@ -import { encode, type PBLink, type PBNode, prepare } from '@ipld/dag-pb' +import { encode, prepare } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import parallelBatch from 'it-parallel-batch' import * as rawCodec from 'multiformats/codecs/raw' @@ -6,6 +6,7 @@ import { CustomProgressEvent } from 'progress-events' import { persist } from '../utils/persist.js' import type { BufferImporter, File, InProgressImportResult, WritableStorage, SingleBlockImportResult, ImporterProgressEvents } from '../index.js' import type { FileLayout, Reducer } from '../layout/index.js' +import type { PBLink, PBNode } from '@ipld/dag-pb' import type { CID, Version } from 'multiformats/cid' import type { ProgressOptions, ProgressEvent } from 'progress-events' diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index b9bfe8f6..33a032b7 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,7 +1,9 @@ import { CustomProgressEvent } from 'progress-events' import { InvalidContentError } from '../errors.js' -import { dirBuilder, type DirBuilderOptions } from './dir.js' -import { fileBuilder, type FileBuilderOptions } from './file.js' +import { dirBuilder } from './dir.js' +import { fileBuilder } from './file.js' +import type { DirBuilderOptions } from './dir.js' +import type { FileBuilderOptions } from './file.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.js' diff --git a/packages/ipfs-unixfs-importer/src/dir-flat.ts b/packages/ipfs-unixfs-importer/src/dir-flat.ts index b649b430..d3548df4 100644 --- a/packages/ipfs-unixfs-importer/src/dir-flat.ts +++ b/packages/ipfs-unixfs-importer/src/dir-flat.ts @@ -1,8 +1,11 @@ -import { encode, type PBNode, prepare } from '@ipld/dag-pb' +import { encode, prepare } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' -import { Dir, CID_V0, CID_V1, type DirProps } from './dir.js' -import { persist, type PersistOptions } from './utils/persist.js' +import { Dir, CID_V0, CID_V1 } from './dir.js' +import { persist } from './utils/persist.js' +import type { DirProps } from './dir.js' import type { ImportResult, InProgressImportResult } from './index.js' +import type { PersistOptions } from './utils/persist.js' +import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' import type { CID } from 'multiformats/cid' @@ -35,11 +38,11 @@ export class DirFlat extends Dir { return this.childCount() } - onlyChild (): InProgressImportResult | Dir { + onlyChild (): InProgressImportResult | Dir | undefined { return this._children.values().next().value } - async * eachChildSeries (): AsyncGenerator<{ key: string, child: InProgressImportResult | Dir }, void, undefined> { + * eachChildSeries (): Generator<{ key: string, child: InProgressImportResult | Dir }, void, undefined> { for (const [key, child] of this._children.entries()) { yield { key, diff --git a/packages/ipfs-unixfs-importer/src/dir-sharded.ts b/packages/ipfs-unixfs-importer/src/dir-sharded.ts index 7bee99b0..4e65849c 100644 --- a/packages/ipfs-unixfs-importer/src/dir-sharded.ts +++ b/packages/ipfs-unixfs-importer/src/dir-sharded.ts @@ -1,10 +1,14 @@ -import { encode, type PBLink, prepare } from '@ipld/dag-pb' +import { encode, prepare } from '@ipld/dag-pb' import { murmur3128 } from '@multiformats/murmur3' -import { createHAMT, Bucket, type BucketChild } from 'hamt-sharding' +import { createHAMT, Bucket } from 'hamt-sharding' import { UnixFS } from 'ipfs-unixfs' -import { Dir, CID_V0, CID_V1, type DirProps } from './dir.js' -import { persist, type PersistOptions } from './utils/persist.js' +import { Dir, CID_V0, CID_V1 } from './dir.js' +import { persist } from './utils/persist.js' +import type { DirProps } from './dir.js' import type { ImportResult, InProgressImportResult } from './index.js' +import type { PersistOptions } from './utils/persist.js' +import type { PBLink } from '@ipld/dag-pb' +import type { BucketChild } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' async function hamtHashFn (buf: Uint8Array): Promise { @@ -60,8 +64,8 @@ class DirSharded extends Dir { return this._bucket.onlyChild() } - async * eachChildSeries (): AsyncGenerator<{ key: string, child: InProgressImportResult | Dir }> { - for await (const { key, value } of this._bucket.eachLeafSeries()) { + * eachChildSeries (): Generator<{ key: string, child: InProgressImportResult | Dir }> { + for (const { key, value } of this._bucket.eachLeafSeries()) { yield { key, child: value diff --git a/packages/ipfs-unixfs-importer/src/dir.ts b/packages/ipfs-unixfs-importer/src/dir.ts index 3e1b2a4c..c13118ca 100644 --- a/packages/ipfs-unixfs-importer/src/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dir.ts @@ -49,7 +49,7 @@ export abstract class Dir { abstract put (name: string, value: InProgressImportResult | Dir): Promise abstract get (name: string): Promise - abstract eachChildSeries (): AsyncIterable<{ key: string, child: InProgressImportResult | Dir }> + abstract eachChildSeries (): Iterable<{ key: string, child: InProgressImportResult | Dir }> abstract flush (blockstore: WritableStorage): AsyncGenerator abstract estimateNodeSize (): number abstract childCount (): number diff --git a/packages/ipfs-unixfs-importer/src/flat-to-shard.ts b/packages/ipfs-unixfs-importer/src/flat-to-shard.ts index f7e58959..fd028366 100644 --- a/packages/ipfs-unixfs-importer/src/flat-to-shard.ts +++ b/packages/ipfs-unixfs-importer/src/flat-to-shard.ts @@ -1,5 +1,6 @@ import { DirFlat } from './dir-flat.js' -import DirSharded, { type DirShardedOptions } from './dir-sharded.js' +import DirSharded from './dir-sharded.js' +import type { DirShardedOptions } from './dir-sharded.js' import type { Dir } from './dir.js' export async function flatToShard (child: Dir | null, dir: Dir, threshold: number, options: DirShardedOptions): Promise { @@ -43,7 +44,7 @@ async function convertToShard (oldDir: DirFlat, options: DirShardedOptions): Pro mode: oldDir.mode }, options) - for await (const { key, child } of oldDir.eachChildSeries()) { + for (const { key, child } of oldDir.eachChildSeries()) { await newDir.put(key, child) } diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 859b877d..8497a79f 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -65,14 +65,18 @@ import first from 'it-first' import parallelBatch from 'it-parallel-batch' import { fixedSize } from './chunker/fixed-size.js' -import { type BufferImportProgressEvents, defaultBufferImporter } from './dag-builder/buffer-importer.js' -import { type DAGBuilder, type DagBuilderProgressEvents, defaultDagBuilder } from './dag-builder/index.js' -import { type ChunkValidator, defaultChunkValidator } from './dag-builder/validate-chunks.js' +import { defaultBufferImporter } from './dag-builder/buffer-importer.js' +import { defaultDagBuilder } from './dag-builder/index.js' +import { defaultChunkValidator } from './dag-builder/validate-chunks.js' import { InvalidParametersError } from './errors.js' -import { balanced, type FileLayout } from './layout/index.js' +import { balanced } from './layout/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { Chunker } from './chunker/index.js' +import type { BufferImportProgressEvents } from './dag-builder/buffer-importer.js' import type { ReducerProgressEvents } from './dag-builder/file.js' +import type { DAGBuilder, DagBuilderProgressEvents } from './dag-builder/index.js' +import type { ChunkValidator } from './dag-builder/validate-chunks.js' +import type { FileLayout } from './layout/index.js' import type { Blockstore } from 'interface-blockstore' import type { AwaitIterable } from 'interface-store' import type { UnixFS, Mtime } from 'ipfs-unixfs' diff --git a/packages/ipfs-unixfs-importer/src/tree-builder.ts b/packages/ipfs-unixfs-importer/src/tree-builder.ts index 18e94eb0..9e9b4fcc 100644 --- a/packages/ipfs-unixfs-importer/src/tree-builder.ts +++ b/packages/ipfs-unixfs-importer/src/tree-builder.ts @@ -112,7 +112,7 @@ export function defaultTreeBuilder (options: TreeBuilderOptions): TreeBuilder { if (options.wrapWithDirectory || (singleRoot && tree.childCount() > 1)) { yield * flushAndYield(tree, block) } else { - for await (const unwrapped of tree.eachChildSeries()) { + for (const unwrapped of tree.eachChildSeries()) { if (unwrapped == null) { continue } diff --git a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.ts b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.ts index 04637910..0dda35f3 100644 --- a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.ts +++ b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.ts @@ -5,9 +5,11 @@ import { MemoryBlockstore } from 'blockstore-core' import first from 'it-first' import last from 'it-last' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { importer, type ImporterOptions } from '../src/index.js' -import { balanced, type FileLayout, flat, trickle } from '../src/layout/index.js' +import { importer } from '../src/index.js' +import { balanced, flat, trickle } from '../src/layout/index.js' import randomByteStream from './helpers/finite-pseudorandom-byte-stream.js' +import type { ImporterOptions } from '../src/index.js' +import type { FileLayout } from '../src/layout/index.js' const strategies: Record<'flat' | 'trickle' | 'balanced', FileLayout> = { flat: flat(), diff --git a/packages/ipfs-unixfs-importer/typedoc.json b/packages/ipfs-unixfs-importer/typedoc.json index 34037848..941072f4 100644 --- a/packages/ipfs-unixfs-importer/typedoc.json +++ b/packages/ipfs-unixfs-importer/typedoc.json @@ -1,4 +1,5 @@ { + "readme": "none", "entryPoints": [ "./src/index.ts", "./src/chunker/index.ts", diff --git a/packages/ipfs-unixfs/CODE_OF_CONDUCT.md b/packages/ipfs-unixfs/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..6b0fa54c --- /dev/null +++ b/packages/ipfs-unixfs/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Contributor Code of Conduct + +This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) diff --git a/packages/ipfs-unixfs/LICENSE b/packages/ipfs-unixfs/LICENSE deleted file mode 100644 index 20ce483c..00000000 --- a/packages/ipfs-unixfs/LICENSE +++ /dev/null @@ -1,4 +0,0 @@ -This project is dual licensed under MIT and Apache-2.0. - -MIT: https://www.opensource.org/licenses/mit -Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/ipfs-unixfs/LICENSE-APACHE b/packages/ipfs-unixfs/LICENSE-APACHE index 14478a3b..b09cd785 100644 --- a/packages/ipfs-unixfs/LICENSE-APACHE +++ b/packages/ipfs-unixfs/LICENSE-APACHE @@ -1,5 +1,201 @@ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index ef24b7c4..fe5784da 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -32,16 +32,6 @@ "import": "./dist/src/index.js" } }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "project": true, - "sourceType": "module" - }, - "ignorePatterns": [ - "src/unixfs.d.ts" - ] - }, "release": { "branches": [ "main" @@ -124,7 +114,15 @@ "@semantic-release/changelog", "@semantic-release/npm", "@semantic-release/github", - "@semantic-release/git" + [ + "@semantic-release/git", + { + "assets": [ + "CHANGELOG.md", + "package.json" + ] + } + ] ] }, "scripts": { @@ -144,8 +142,8 @@ "uint8arraylist": "^2.4.8" }, "devDependencies": { - "aegir": "^45.0.8", - "protons": "^7.6.0", + "aegir": "^47.0.16", + "protons": "^7.6.1", "uint8arrays": "^5.1.0" }, "browser": { diff --git a/packages/ipfs-unixfs/src/unixfs.ts b/packages/ipfs-unixfs/src/unixfs.ts index ceed9332..d6feac8d 100644 --- a/packages/ipfs-unixfs/src/unixfs.ts +++ b/packages/ipfs-unixfs/src/unixfs.ts @@ -1,9 +1,3 @@ -/* eslint-disable import/export */ -/* eslint-disable complexity */ -/* eslint-disable @typescript-eslint/no-namespace */ -/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */ -/* eslint-disable @typescript-eslint/no-empty-interface */ - import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime' import type { Codec } from 'protons-runtime' import type { Uint8ArrayList } from 'uint8arraylist' diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.ts b/packages/ipfs-unixfs/test/unixfs-format.spec.ts index 12dae55b..15c76373 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.ts +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.ts @@ -3,8 +3,9 @@ import { expect } from 'aegir/chai' import loadFixture from 'aegir/fixtures' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { type Mtime, UnixFS } from '../src/index.js' +import { UnixFS } from '../src/index.js' import * as Pb from '../src/unixfs.js' +import type { Mtime } from '../src/index.js' const PBData = Pb.Data const raw = loadFixture('test/fixtures/raw.unixfs') diff --git a/packages/ipfs-unixfs/typedoc.json b/packages/ipfs-unixfs/typedoc.json index f599dc72..db0b0747 100644 --- a/packages/ipfs-unixfs/typedoc.json +++ b/packages/ipfs-unixfs/typedoc.json @@ -1,4 +1,5 @@ { + "readme": "none", "entryPoints": [ "./src/index.ts" ] diff --git a/typedoc.json b/typedoc.json index 481c04ce..a7a0c07b 100644 --- a/typedoc.json +++ b/typedoc.json @@ -1,7 +1,12 @@ { "$schema": "https://typedoc.org/schema.json", - "name": "UnixFS", + "name": "js-ipfs-unixfs", + "readme": "./README.md", + "headings": { + "readme": false, + "document": false + }, "exclude": [ - "packages/interop" + "benchmarks/**/*" ] } From 2fe23fe9c2320fca660fe611c90c6bf432196cf4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:30:55 +0000 Subject: [PATCH 22/42] chore(release): 11.2.3 [skip ci] ## [ipfs-unixfs-v11.2.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.2...ipfs-unixfs-11.2.3) (2025-06-18) ### Dependencies * bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 956b1de1..61f2e2e0 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.2...ipfs-unixfs-11.2.3) (2025-06-18) + +### Dependencies + +* bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) + ## [ipfs-unixfs-v11.2.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.1...ipfs-unixfs-11.2.2) (2025-06-16) ### Bug Fixes diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index fe5784da..f0be6837 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.2", + "version": "11.2.3", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From 2c1c244c5dfdabe9b13e2551f70bc1bdf0c60f29 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:31:12 +0000 Subject: [PATCH 23/42] chore(release): 15.3.3 [skip ci] ## [ipfs-unixfs-importer-v15.3.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.2...ipfs-unixfs-importer-15.3.3) (2025-06-18) ### Dependencies * bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 8033da38..bbcf3bc2 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.3.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.2...ipfs-unixfs-importer-15.3.3) (2025-06-18) + +### Dependencies + +* bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) + ## [ipfs-unixfs-importer-v15.3.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.1...ipfs-unixfs-importer-15.3.2) (2025-03-06) ### Dependencies diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 0511a28e..ffb17816 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.2", + "version": "15.3.3", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From ad084ec0cfe69c19bca3a6a61f01bdace20bdade Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:31:36 +0000 Subject: [PATCH 24/42] chore(release): 13.6.4 [skip ci] ## [ipfs-unixfs-exporter-v13.6.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.3...ipfs-unixfs-exporter-13.6.4) (2025-06-18) ### Dependencies * bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 3d046c17..5000efbf 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.3...ipfs-unixfs-exporter-13.6.4) (2025-06-18) + +### Dependencies + +* bump aegir from 45.2.1 to 47.0.16 ([#431](https://github.com/ipfs/js-ipfs-unixfs/issues/431)) ([1fb2db3](https://github.com/ipfs/js-ipfs-unixfs/commit/1fb2db37f33674eb7a0e00aa88d5312a7644536d)) + ## [ipfs-unixfs-exporter-v13.6.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.2...ipfs-unixfs-exporter-13.6.3) (2025-06-16) ### Dependencies diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index f77b4eab..9304eba5 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.3", + "version": "13.6.4", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 95e0b47de62c57b29bd10d48503cef4f208caae1 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 18 Jun 2025 07:48:52 +0300 Subject: [PATCH 25/42] docs: convert examples to ts, run doc verifier (#434) Enables type checking for docs so they are kept in sync with API changes. --- package.json | 1 + packages/ipfs-unixfs-exporter/README.md | 10 +++- packages/ipfs-unixfs-exporter/package.json | 1 + packages/ipfs-unixfs-exporter/src/index.ts | 22 +++++--- packages/ipfs-unixfs-importer/README.md | 4 +- packages/ipfs-unixfs-importer/package.json | 1 + packages/ipfs-unixfs-importer/src/index.ts | 4 +- packages/ipfs-unixfs/README.md | 65 +++++++++++++++------- packages/ipfs-unixfs/package.json | 1 + packages/ipfs-unixfs/src/index.ts | 65 +++++++++++++++------- 10 files changed, 120 insertions(+), 54 deletions(-) diff --git a/package.json b/package.json index 13c17a6f..ab2f7c5d 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "build": "aegir run build", "lint": "aegir run lint", "dep-check": "aegir run dep-check", + "doc-check": "aegir run doc-check", "release": "run-s build docs:no-publish npm:release docs", "npm:release": "aegir run release --concurrency 1", "docs": "aegir docs", diff --git a/packages/ipfs-unixfs-exporter/README.md b/packages/ipfs-unixfs-exporter/README.md index 899c2d81..19949f4a 100644 --- a/packages/ipfs-unixfs-exporter/README.md +++ b/packages/ipfs-unixfs-exporter/README.md @@ -28,7 +28,7 @@ The UnixFS Exporter provides a means to read DAGs from a blockstore given a CID. ## Example -```js +```TypeScript // import a file and export it again import { importer } from 'ipfs-unixfs-importer' import { exporter } from 'ipfs-unixfs-exporter' @@ -49,6 +49,10 @@ console.info(files[0].cid) // Qmbaz const entry = await exporter(files[0].cid, blockstore) +if (entry.type !== 'file') { + throw new Error('Unexpected entry type') +} + console.info(entry.cid) // Qmqux console.info(entry.path) // Qmbaz/foo/bar.txt console.info(entry.name) // bar.txt @@ -56,12 +60,12 @@ console.info(entry.unixfs.fileSize()) // 4 // stream content from unixfs node const size = entry.unixfs.fileSize() -const bytes = new Uint8Array(size) +const bytes = new Uint8Array(Number(size)) let offset = 0 for await (const buf of entry.content()) { bytes.set(buf, offset) - offset += chunk.length + offset += buf.byteLength } console.info(bytes) // 0, 1, 2, 3 diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 9304eba5..127fd8d8 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -134,6 +134,7 @@ "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check", + "doc-check": "aegir doc-check", "release": "aegir release" }, "dependencies": { diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index ead6279a..e8b06f59 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -5,7 +5,7 @@ * * @example * - * ```js + * ```TypeScript * // import a file and export it again * import { importer } from 'ipfs-unixfs-importer' * import { exporter } from 'ipfs-unixfs-exporter' @@ -26,6 +26,10 @@ * * const entry = await exporter(files[0].cid, blockstore) * + * if (entry.type !== 'file') { + * throw new Error('Unexpected entry type') + * } + * * console.info(entry.cid) // Qmqux * console.info(entry.path) // Qmbaz/foo/bar.txt * console.info(entry.name) // bar.txt @@ -33,12 +37,12 @@ * * // stream content from unixfs node * const size = entry.unixfs.fileSize() - * const bytes = new Uint8Array(size) + * const bytes = new Uint8Array(Number(size)) * let offset = 0 * * for await (const buf of entry.content()) { * bytes.set(buf, offset) - * offset += chunk.length + * offset += buf.byteLength * } * * console.info(bytes) // 0, 1, 2, 3 @@ -180,7 +184,7 @@ export interface Exportable { * * When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data: * - * ```javascript + * ```TypeScript * const length = 5 * const data = new Uint8Array(length) * let offset = 0 @@ -201,7 +205,7 @@ export interface Exportable { * * If `entry` is a directory, passing `offset` and/or `length` to `entry.content()` will limit the number of files returned from the directory. * - * ```javascript + * ```TypeScript * const entries = [] * * for await (const entry of dir.content({ @@ -220,7 +224,7 @@ export interface Exportable { /** * If the entry is a file, `entry.content()` returns an async iterator that yields one or more Uint8Arrays containing the file content: * - * ```javascript + * ```TypeScript * if (entry.type === 'file') { * for await (const chunk of entry.content()) { * // chunk is a Buffer @@ -237,7 +241,7 @@ export interface UnixFSFile extends Exportable { /** * If the entry is a directory, `entry.content()` returns further `entry` objects: * - * ```javascript + * ```TypeScript * if (entry.type === 'directory') { * for await (const entry of dir.content()) { * console.info(entry.name) @@ -264,7 +268,7 @@ export interface ObjectNode extends Exportable { * * `entry.content()` returns an async iterator that yields a buffer containing the node content: * - * ```javascript + * ```TypeScript * for await (const chunk of entry.content()) { * // chunk is a Buffer * } @@ -371,7 +375,7 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * * @example * - * ```javascript + * ```TypeScript * import { walkPath } from 'ipfs-unixfs-exporter' * * const entries = [] diff --git a/packages/ipfs-unixfs-importer/README.md b/packages/ipfs-unixfs-importer/README.md index c0f69b4a..05c5c174 100644 --- a/packages/ipfs-unixfs-importer/README.md +++ b/packages/ipfs-unixfs-importer/README.md @@ -37,7 +37,7 @@ Let's create a little directory to import: And write the importing logic: -```js +```TypeScript import { importer } from 'ipfs-unixfs-importer' import { MemoryBlockstore } from 'blockstore-core/memory' import * as fs from 'node:fs' @@ -61,7 +61,7 @@ for await (const entry of importer(source, blockstore)) { When run, metadata about DAGNodes in the created tree is printed until the root: -```js +``` { cid: CID, // see https://github.com/multiformats/js-cid path: 'tmp/foo/bar', diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index ffb17816..925ade49 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -158,6 +158,7 @@ "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check", + "doc-check": "aegir doc-check", "release": "aegir release" }, "dependencies": { diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 8497a79f..84e22564 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -14,7 +14,7 @@ * * And write the importing logic: * - * ```js + * ```TypeScript * import { importer } from 'ipfs-unixfs-importer' * import { MemoryBlockstore } from 'blockstore-core/memory' * import * as fs from 'node:fs' @@ -38,7 +38,7 @@ * * When run, metadata about DAGNodes in the created tree is printed until the root: * - * ```js + * ``` * { * cid: CID, // see https://github.com/multiformats/js-cid * path: 'tmp/foo/bar', diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index 52198d1f..d8757440 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -30,10 +30,12 @@ The UnixFS spec can be found in the [ipfs/specs repository](http://github.com/ip ## Example - Create a file composed of several blocks -```JavaScript +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + const data = new UnixFS({ type: 'file' }) -data.addBlockSize(256) // add the size of each block -data.addBlockSize(256) +data.addBlockSize(256n) // add the size of each block +data.addBlockSize(256n) // ... ``` @@ -41,14 +43,20 @@ data.addBlockSize(256) Creating a directory that contains several files is achieve by creating a unixfs element that identifies a MerkleDAG node as a directory. The links of that MerkleDAG node are the files that are contained in this directory. -```JavaScript +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + const data = new UnixFS({ type: 'directory' }) ``` ## Example - Create an unixfs Data element -```JavaScript -const data = new UnixFS([options]) +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const data = new UnixFS({ + // ...options +}) ``` `options` is an optional object argument that might include the following keys: @@ -67,34 +75,51 @@ const data = new UnixFS([options]) ## Example - Add and remove a block size to the block size list -```JavaScript -data.addBlockSize() +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const data = new UnixFS({ type: 'file' }) +const sizeInBytes = 100n +data.addBlockSize(sizeInBytes) ``` -```JavaScript -data.removeBlockSize() +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const data = new UnixFS({ type: 'file' }) + +const index = 0 +data.removeBlockSize(index) ``` ## Example - Get total fileSize -```JavaScript +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const data = new UnixFS({ type: 'file' }) data.fileSize() // => size in bytes ``` ## Example - Marshal and unmarshal -```javascript +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const data = new UnixFS({ type: 'file' }) const marshaled = data.marshal() -const unmarshaled = Unixfs.unmarshal(marshaled) +const unmarshaled = UnixFS.unmarshal(marshaled) ``` ## Example - Is this UnixFS entry a directory? -```JavaScript -const dir = new Data({ type: 'directory' }) +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const dir = new UnixFS({ type: 'directory' }) dir.isDirectory() // true -const file = new Data({ type: 'file' }) +const file = new UnixFS({ type: 'file' }) file.isDirectory() // false ``` @@ -102,13 +127,15 @@ file.isDirectory() // false If no modification time has been set, no `mtime` property will be present on the `Data` instance: -```JavaScript -const file = new Data({ type: 'file' }) +```TypeScript +import { UnixFS } from 'ipfs-unixfs' + +const file = new UnixFS({ type: 'file' }) file.mtime // undefined Object.prototype.hasOwnProperty.call(file, 'mtime') // false -const dir = new Data({ type: 'dir', mtime: new Date() }) +const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) dir.mtime // { secs: Number, nsecs: Number } ``` diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index f0be6837..d98fc6a8 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -135,6 +135,7 @@ "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check", + "doc-check": "aegir doc-check", "release": "aegir release" }, "dependencies": { diff --git a/packages/ipfs-unixfs/src/index.ts b/packages/ipfs-unixfs/src/index.ts index 82f76823..a5a98ed9 100644 --- a/packages/ipfs-unixfs/src/index.ts +++ b/packages/ipfs-unixfs/src/index.ts @@ -7,10 +7,12 @@ * * @example Create a file composed of several blocks * - * ```JavaScript + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * * const data = new UnixFS({ type: 'file' }) - * data.addBlockSize(256) // add the size of each block - * data.addBlockSize(256) + * data.addBlockSize(256n) // add the size of each block + * data.addBlockSize(256n) * // ... * ``` * @@ -18,14 +20,20 @@ * * Creating a directory that contains several files is achieve by creating a unixfs element that identifies a MerkleDAG node as a directory. The links of that MerkleDAG node are the files that are contained in this directory. * - * ```JavaScript + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * * const data = new UnixFS({ type: 'directory' }) * ``` * * @example Create an unixfs Data element * - * ```JavaScript - * const data = new UnixFS([options]) + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const data = new UnixFS({ + * // ...options + * }) * ``` * * `options` is an optional object argument that might include the following keys: @@ -44,34 +52,51 @@ * * @example Add and remove a block size to the block size list * - * ```JavaScript - * data.addBlockSize() + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const data = new UnixFS({ type: 'file' }) + * const sizeInBytes = 100n + * data.addBlockSize(sizeInBytes) * ``` * - * ```JavaScript - * data.removeBlockSize() + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const data = new UnixFS({ type: 'file' }) + * + * const index = 0 + * data.removeBlockSize(index) * ``` * * @example Get total fileSize * - * ```JavaScript + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const data = new UnixFS({ type: 'file' }) * data.fileSize() // => size in bytes * ``` * * @example Marshal and unmarshal * - * ```javascript + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const data = new UnixFS({ type: 'file' }) * const marshaled = data.marshal() - * const unmarshaled = Unixfs.unmarshal(marshaled) + * const unmarshaled = UnixFS.unmarshal(marshaled) * ``` * * @example Is this UnixFS entry a directory? * - * ```JavaScript - * const dir = new Data({ type: 'directory' }) + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const dir = new UnixFS({ type: 'directory' }) * dir.isDirectory() // true * - * const file = new Data({ type: 'file' }) + * const file = new UnixFS({ type: 'file' }) * file.isDirectory() // false * ``` * @@ -79,13 +104,15 @@ * * If no modification time has been set, no `mtime` property will be present on the `Data` instance: * - * ```JavaScript - * const file = new Data({ type: 'file' }) + * ```TypeScript + * import { UnixFS } from 'ipfs-unixfs' + * + * const file = new UnixFS({ type: 'file' }) * file.mtime // undefined * * Object.prototype.hasOwnProperty.call(file, 'mtime') // false * - * const dir = new Data({ type: 'dir', mtime: new Date() }) + * const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) * dir.mtime // { secs: Number, nsecs: Number } * ``` */ From 8885871e6437797fb2d2cc6049e0edaf60fb1025 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:54:46 +0000 Subject: [PATCH 26/42] chore(release): 11.2.4 [skip ci] ## [ipfs-unixfs-v11.2.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.3...ipfs-unixfs-11.2.4) (2025-06-18) ### Documentation * convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 61f2e2e0..5f20200d 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.3...ipfs-unixfs-11.2.4) (2025-06-18) + +### Documentation + +* convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) + ## [ipfs-unixfs-v11.2.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.2...ipfs-unixfs-11.2.3) (2025-06-18) ### Dependencies diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index d98fc6a8..6a81edef 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.3", + "version": "11.2.4", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From 9bde19acf047ec9106229eaebd1c4af3a46071ec Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:55:04 +0000 Subject: [PATCH 27/42] chore(release): 15.3.4 [skip ci] ## [ipfs-unixfs-importer-v15.3.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.3...ipfs-unixfs-importer-15.3.4) (2025-06-18) ### Documentation * convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index bbcf3bc2..4ca31531 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.3.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.3...ipfs-unixfs-importer-15.3.4) (2025-06-18) + +### Documentation + +* convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) + ## [ipfs-unixfs-importer-v15.3.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.2...ipfs-unixfs-importer-15.3.3) (2025-06-18) ### Dependencies diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 925ade49..4eda1c59 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.3", + "version": "15.3.4", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From 28a9c686e65195a59a5a6c5c0d3860e85d9a8982 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 04:55:21 +0000 Subject: [PATCH 28/42] chore(release): 13.6.5 [skip ci] ## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) ### Documentation * convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 5000efbf..f11662da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) + +### Documentation + +* convert examples to ts, run doc verifier ([#434](https://github.com/ipfs/js-ipfs-unixfs/issues/434)) ([95e0b47](https://github.com/ipfs/js-ipfs-unixfs/commit/95e0b47de62c57b29bd10d48503cef4f208caae1)) + ## [ipfs-unixfs-exporter-v13.6.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.3...ipfs-unixfs-exporter-13.6.4) (2025-06-18) ### Dependencies diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 127fd8d8..2110db9e 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.4", + "version": "13.6.5", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From d06d9fe5aa4c0e7f82f3265bf09b0db064d2b563 Mon Sep 17 00:00:00 2001 From: clostao <49268919+clostao@users.noreply.github.com> Date: Wed, 18 Jun 2025 07:55:35 +0200 Subject: [PATCH 29/42] feat: enable custom file/directory builders (#413) To allow adding building custom file/directory IPLD objects, two additional fields to `DagBuilderOptions` & `ImporterOptions` are added that would override the `dirBuilder` & `fileBuilder` with other implementations. --------- Co-authored-by: achingbrain --- .../src/dag-builder/dir.ts | 6 ++- .../src/dag-builder/file.ts | 15 +++++-- .../src/dag-builder/index.ts | 18 ++++++--- packages/ipfs-unixfs-importer/src/index.ts | 21 +++++++++- .../test/custom-dag-builder-params.spec.ts | 40 +++++++++++++++++++ 5 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts index a29675f6..06996f46 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts @@ -9,7 +9,11 @@ export interface DirBuilderOptions { signal?: AbortSignal } -export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { +export interface DirBuilder { + (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise +} + +export const defaultDirBuilder: DirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { const unixfs = new UnixFS({ type: 'directory', mtime: dir.mtime, diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 41862bce..db359606 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -30,7 +30,7 @@ async function * buildFileBatch (file: File, blockstore: WritableStorage, option } continue - } else if (count === 1 && (previous != null)) { + } else if (count === 1 && previous != null) { // we have the second block of a multiple block import so yield the first yield { ...previous, @@ -131,7 +131,7 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return true } - if ((leaf.unixfs != null) && (leaf.unixfs.data == null) && leaf.unixfs.fileSize() > 0n) { + if (leaf.unixfs != null && leaf.unixfs.data == null && leaf.unixfs.fileSize() > 0n) { return true } @@ -189,10 +189,17 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return reducer } +export interface FileBuilder { + (file: File, blockstore: WritableStorage, options: FileBuilderOptions): Promise +} + export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions { layout: FileLayout } -export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { - return options.layout(buildFileBatch(file, block, options), reduce(file, block, options)) +export const defaultFileBuilder: FileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { + return options.layout( + buildFileBatch(file, block, options), + reduce(file, block, options) + ) } diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 33a032b7..475c3515 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,9 +1,9 @@ import { CustomProgressEvent } from 'progress-events' import { InvalidContentError } from '../errors.js' -import { dirBuilder } from './dir.js' -import { fileBuilder } from './file.js' -import type { DirBuilderOptions } from './dir.js' -import type { FileBuilderOptions } from './file.js' +import { defaultDirBuilder } from './dir.js' +import { defaultFileBuilder } from './file.js' +import type { DirBuilder, DirBuilderOptions } from './dir.js' +import type { FileBuilder, FileBuilderOptions } from './file.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.js' @@ -45,11 +45,11 @@ function contentAsAsyncIterable (content: Uint8Array | AsyncIterable if (content instanceof Uint8Array) { return (async function * () { yield content - }()) + })() } else if (isIterable(content)) { return (async function * () { yield * content - }()) + })() } else if (isAsyncIterable(content)) { return content } @@ -64,6 +64,8 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions chunker: Chunker chunkValidator: ChunkValidator wrapWithDirectory: boolean + dirBuilder?: DirBuilder + fileBuilder?: FileBuilder } export type ImporterSourceStream = AsyncIterable | Iterable @@ -109,6 +111,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const fileBuilder = options.fileBuilder ?? defaultFileBuilder + yield async () => fileBuilder(file, blockstore, options) } else if (entry.path != null) { const dir: Directory = { @@ -118,6 +122,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const dirBuilder = options.dirBuilder ?? defaultDirBuilder + yield async () => dirBuilder(dir, blockstore, options) } else { throw new Error('Import candidate must have content or path or both') diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 84e22564..2f00975f 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -73,7 +73,8 @@ import { balanced } from './layout/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { Chunker } from './chunker/index.js' import type { BufferImportProgressEvents } from './dag-builder/buffer-importer.js' -import type { ReducerProgressEvents } from './dag-builder/file.js' +import type { DirBuilder } from './dag-builder/dir.js' +import type { FileBuilder, ReducerProgressEvents } from './dag-builder/file.js' import type { DAGBuilder, DagBuilderProgressEvents } from './dag-builder/index.js' import type { ChunkValidator } from './dag-builder/validate-chunks.js' import type { FileLayout } from './layout/index.js' @@ -276,6 +277,20 @@ export interface ImporterOptions extends ProgressOptions * `Error` */ chunkValidator?: ChunkValidator + + /** + * This option can be used to override how a directory IPLD node is built. + * + * This function takes a `Directory` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + dirBuilder?: DirBuilder + + /** + * This option can be used to override how a file IPLD node is built. + * + * This function takes a `File` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + fileBuilder?: FileBuilder } export type ImportCandidateStream = AsyncIterable | Iterable @@ -342,7 +357,9 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri blockWriteConcurrency, reduceSingleLeafToSelf, cidVersion, - onProgress: options.onProgress + onProgress: options.onProgress, + dirBuilder: options.dirBuilder, + fileBuilder: options.fileBuilder }) const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({ wrapWithDirectory, diff --git a/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts new file mode 100644 index 00000000..605e26fc --- /dev/null +++ b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts @@ -0,0 +1,40 @@ +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { defaultDirBuilder } from '../src/dag-builder/dir.js' +import { defaultFileBuilder } from '../src/dag-builder/file.js' +import { importer } from '../src/index.js' +import type { DirBuilder } from '../src/dag-builder/dir.js' +import type { FileBuilder } from '../src/dag-builder/file.js' + +describe('CustomParamsDagBuilder', () => { + it('should build a dag with custom dir builder', async () => { + const counter = { dirCounter: 0, fileCounter: 0 } + const customDirBuilder: DirBuilder = async (...args) => { + counter.dirCounter++ + return defaultDirBuilder(...args) + } + + const customFileBuilder: FileBuilder = async (...args) => { + counter.fileCounter++ + return defaultFileBuilder(...args) + } + + const blockstore = new MemoryBlockstore() + const files = [] + for await (const file of importer([{ + path: './src/file.txt', + content: new Uint8Array( + 'hello world'.split('').map((char) => char.charCodeAt(0)) + ) + }, { + path: './src' + }], blockstore, { + dirBuilder: customDirBuilder, + fileBuilder: customFileBuilder + })) { + files.push(file) + } + + expect(counter.dirCounter).to.equal(1) + }) +}) From b366833ee583f423dfb44cadb79eca5a6ab743c9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 06:01:28 +0000 Subject: [PATCH 30/42] chore(release): 15.4.0 [skip ci] ## [ipfs-unixfs-importer-v15.4.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.4...ipfs-unixfs-importer-15.4.0) (2025-06-18) ### Features * enable custom file/directory builders ([#413](https://github.com/ipfs/js-ipfs-unixfs/issues/413)) ([d06d9fe](https://github.com/ipfs/js-ipfs-unixfs/commit/d06d9fe5aa4c0e7f82f3265bf09b0db064d2b563)) --- packages/ipfs-unixfs-importer/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-importer/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index 4ca31531..98e3b825 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-importer-v15.4.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.4...ipfs-unixfs-importer-15.4.0) (2025-06-18) + +### Features + +* enable custom file/directory builders ([#413](https://github.com/ipfs/js-ipfs-unixfs/issues/413)) ([d06d9fe](https://github.com/ipfs/js-ipfs-unixfs/commit/d06d9fe5aa4c0e7f82f3265bf09b0db064d2b563)) + ## [ipfs-unixfs-importer-v15.3.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-15.3.3...ipfs-unixfs-importer-15.3.4) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 4eda1c59..ac50610b 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "15.3.4", + "version": "15.4.0", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-importer#readme", From 7663b87ed2e3e8cd4da1484ca601638740ea0ae7 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 18 Jun 2025 11:02:42 +0300 Subject: [PATCH 31/42] fix: constrain the unixfs type (#435) Turns the unixfs type into a proper type to enable code completion and type checking. --- packages/ipfs-unixfs-exporter/test/exporter.spec.ts | 3 ++- packages/ipfs-unixfs/README.md | 2 +- packages/ipfs-unixfs/src/index.ts | 8 +++++--- packages/ipfs-unixfs/test/unixfs-format.spec.ts | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index a48fe9bb..67326ec4 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -31,6 +31,7 @@ import { exporter, recursive } from '../src/index.js' import asAsyncIterable from './helpers/as-async-iterable.js' import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' +import type { UnixFSType } from 'ipfs-unixfs' import type { Chunker } from 'ipfs-unixfs-importer/chunker' import type { FileLayout } from 'ipfs-unixfs-importer/layout' @@ -46,7 +47,7 @@ describe('exporter', () => { smallFile = uint8ArrayConcat(await all(randomBytes(200))) }) - async function dagPut (options: { type?: string, content?: Uint8Array, links?: dagPb.PBLink[] } = {}): Promise<{ file: UnixFS, node: PBNode, cid: CID }> { + async function dagPut (options: { type?: UnixFSType, content?: Uint8Array, links?: dagPb.PBLink[] } = {}): Promise<{ file: UnixFS, node: PBNode, cid: CID }> { options.type = options.type ?? 'file' options.content = options.content ?? Uint8Array.from([0x01, 0x02, 0x03]) options.links = options.links ?? [] diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index d8757440..aa23bcb0 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -135,7 +135,7 @@ file.mtime // undefined Object.prototype.hasOwnProperty.call(file, 'mtime') // false -const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) +const dir = new UnixFS({ type: 'directory', mtime: { secs: 5n } }) dir.mtime // { secs: Number, nsecs: Number } ``` diff --git a/packages/ipfs-unixfs/src/index.ts b/packages/ipfs-unixfs/src/index.ts index a5a98ed9..04e03973 100644 --- a/packages/ipfs-unixfs/src/index.ts +++ b/packages/ipfs-unixfs/src/index.ts @@ -112,7 +112,7 @@ * * Object.prototype.hasOwnProperty.call(file, 'mtime') // false * - * const dir = new UnixFS({ type: 'dir', mtime: { secs: 5n } }) + * const dir = new UnixFS({ type: 'directory', mtime: { secs: 5n } }) * dir.mtime // { secs: Number, nsecs: Number } * ``` */ @@ -127,7 +127,9 @@ export interface Mtime { export type MtimeLike = Mtime | { Seconds: number, FractionalNanoseconds?: number } | [number, number] | Date -const types: Record = { +export type UnixFSType = 'raw' | 'directory' | 'file' | 'metadata' | 'symlink' | 'hamt-sharded-directory' + +const types: Record = { Raw: 'raw', Directory: 'directory', File: 'file', @@ -148,7 +150,7 @@ const DEFAULT_DIRECTORY_MODE = parseInt('0755', 8) const MAX_FANOUT = BigInt(1 << 10) export interface UnixFSOptions { - type?: string + type?: UnixFSType data?: Uint8Array blockSizes?: bigint[] hashType?: bigint diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.ts b/packages/ipfs-unixfs/test/unixfs-format.spec.ts index 15c76373..0aba16d3 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.ts +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.ts @@ -370,6 +370,7 @@ describe('unixfs-format', () => { try { // eslint-disable-next-line no-new new UnixFS({ + // @ts-expect-error invalid type type: 'bananas' }) } catch (err: any) { From b23c59afd49d6f3a35c8032ec181c353e350574e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 08:09:26 +0000 Subject: [PATCH 32/42] chore(release): 11.2.5 [skip ci] ## [ipfs-unixfs-v11.2.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.4...ipfs-unixfs-11.2.5) (2025-06-18) ### Bug Fixes * constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) --- packages/ipfs-unixfs/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 5f20200d..f5930d07 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-v11.2.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.4...ipfs-unixfs-11.2.5) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-v11.2.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-11.2.3...ipfs-unixfs-11.2.4) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 6a81edef..fbace349 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "11.2.4", + "version": "11.2.5", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs#readme", From ce859ab474e81ecea63ab90533df308b0e4e2624 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 18 Jun 2025 08:09:47 +0000 Subject: [PATCH 33/42] chore(release): 13.6.6 [skip ci] ## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) ### Bug Fixes * constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index f11662da..77395b0d 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) + +### Bug Fixes + +* constrain the unixfs type ([#435](https://github.com/ipfs/js-ipfs-unixfs/issues/435)) ([7663b87](https://github.com/ipfs/js-ipfs-unixfs/commit/7663b87ed2e3e8cd4da1484ca601638740ea0ae7)) + ## [ipfs-unixfs-exporter-v13.6.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.4...ipfs-unixfs-exporter-13.6.5) (2025-06-18) ### Documentation diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 2110db9e..7ec27009 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.5", + "version": "13.6.6", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 93cb3d06713c876ad861fe61082e21af4f1905ac Mon Sep 17 00:00:00 2001 From: web3-bot <81333946+web3-bot@users.noreply.github.com> Date: Sat, 26 Jul 2025 08:29:51 +0100 Subject: [PATCH 34/42] chore: add or force update .github/workflows/stale.yml (#436) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1f9d6b8c..7c955c41 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: Close and mark stale issue +name: Close Stale Issues on: schedule: From 332a794227f7792e1ddee1b1e47d01fd510d6cf4 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 30 Jul 2025 10:06:06 +0200 Subject: [PATCH 35/42] feat: add 'extended' option to exporter (#437) To list directories without resolving the root node of each directory entry, add an `extended` option to the exporter. Defaults to `true` to preserve backwards compatibility. --- packages/ipfs-unixfs-exporter/src/index.ts | 52 ++++++++++++++++++- .../resolvers/unixfs-v1/content/directory.ts | 25 +++++++-- .../content/hamt-sharded-directory.ts | 34 +++++++++--- .../src/utils/is-basic-exporter-options.ts | 5 ++ .../test/exporter-sharded.spec.ts | 52 +++++++++++++++++++ .../test/exporter.spec.ts | 50 ++++++++++++++++++ 6 files changed, 206 insertions(+), 12 deletions(-) create mode 100644 packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index e8b06f59..ee6c2e9e 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -57,6 +57,7 @@ import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' +import type { AbortOptions } from 'it-pushable' import type { ProgressOptions, ProgressEvent } from 'progress-events' export * from './errors.js' @@ -136,6 +137,21 @@ export interface ExporterOptions extends ProgressOptions blockReadConcurrency?: number } +export interface BasicExporterOptions extends ExporterOptions { + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended: false +} + export interface Exportable { /** * A disambiguator to allow TypeScript to work out the type of the entry. @@ -218,7 +234,7 @@ export interface Exportable { * // `entries` contains the first 5 files/directories in the directory * ``` */ - content(options?: ExporterOptions): AsyncGenerator + content(options?: ExporterOptions | BasicExporterOptions): AsyncGenerator } /** @@ -316,7 +332,39 @@ export interface Resolver { (cid: CID, name: string, path: string, toResolve: st export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content } + +export interface UnixfsV1BasicContent { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID + + /** + * Resolve the root node of the entry to parse the UnixFS metadata contained + * there. The metadata will contain what kind of node it is (e.g. file, + * directory, etc), the file size, and more. + */ + resolve(options?: AbortOptions): Promise +} + +export interface UnixFsV1ContentResolver { + (options: ExporterOptions): UnixfsV1Content + (options: BasicExporterOptions): UnixfsV1BasicContent +} + +export interface UnixfsV1Resolver { + (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content +} export interface ShardTraversalContext { hamtDepth: number diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index afab2634..614b33ca 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -3,10 +3,11 @@ import map from 'it-map' import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' -import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { const offset = options.offset ?? 0 const length = options.length ?? node.Links.length const links = node.Links.slice(offset, length) @@ -21,8 +22,24 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de return async () => { const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` - const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) - return result.entry + + const load = async (options = {}): Promise => { + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) + return result.entry + } + + if (isBasicExporterOptions(options)) { + const basic: UnixfsV1BasicContent = { + cid: link.Hash, + name: linkName, + path: linkPath, + resolve: load + } + + return basic + } + + return load(options) } }), source => parallel(source, { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index b08255a2..a3f56189 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -5,11 +5,12 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js' +import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { - function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent { + function yieldHamtDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): UnixfsV1DirectoryContent { options.onProgress?.(new CustomProgressEvent('unixfs:exporter:walk:hamt-sharded-directory', { cid })) @@ -20,7 +21,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, return yieldHamtDirectoryContent } -async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent { +async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): any { const links = node.Links if (node.Data == null) { @@ -47,9 +48,28 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de const name = link.Name != null ? link.Name.substring(padLength) : null if (name != null && name !== '') { - const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options) + const linkPath = `${path}/${name}` + const load = async (options = {}): Promise => { + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + return result.entry + } - return { entries: result.entry == null ? [] : [result.entry] } + if (isBasicExporterOptions(options)) { + return { + entries: [{ + cid: link.Hash, + name, + path: linkPath, + resolve: load + }] + } + } + + return { + entries: [ + await load() + ].filter(Boolean) + } } else { // descend into subshard const block = await blockstore.get(link.Hash, options) @@ -59,7 +79,9 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de cid: link.Hash })) - return { entries: listDirectory(node, path, resolve, depth, blockstore, options) } + return { + entries: listDirectory(node, path, resolve, depth, blockstore, options) + } } } }), diff --git a/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts new file mode 100644 index 00000000..95190ea5 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/utils/is-basic-exporter-options.ts @@ -0,0 +1,5 @@ +import type { BasicExporterOptions } from '../index.js' + +export function isBasicExporterOptions (obj?: any): obj is BasicExporterOptions { + return obj?.extended === false +} diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index dd2354c5..fe16201e 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -363,4 +363,56 @@ describe('exporter sharded', function () { content: file?.node }]).to.deep.equal(files) }) + + it('exports basic sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + expect(dirFile).to.have.property('resolve') + + // should fail because we have deleted this block + await expect(dirFile.resolve()).to.eventually.be.rejected() + } + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 67326ec4..9ce3056e 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1605,4 +1605,54 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') + } + + const exported = await exporter(dirCid, block) + const dirFiles = await all(exported.content()) + + // delete shard contents + for (const entry of dirFiles) { + await block.delete(entry.cid) + } + + // list the contents again, this time just the basic version + const basicDirFiles = await all(exported.content({ + extended: false + })) + expect(basicDirFiles.length).to.equal(dirFiles.length) + + for (let i = 0; i < basicDirFiles.length; i++) { + const dirFile = basicDirFiles[i] + + expect(dirFile).to.have.property('name') + expect(dirFile).to.have.property('path') + expect(dirFile).to.have.property('cid') + expect(dirFile).to.have.property('resolve') + + // should fail because we have deleted this block + await expect(dirFile.resolve()).to.eventually.be.rejected() + } + }) }) From 5a916ab1c291e0091d76fb1093916098fd1441c4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 30 Jul 2025 08:12:34 +0000 Subject: [PATCH 36/42] chore(release): 13.7.0 [skip ci] ## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) ### Features * add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 77395b0d..bc190919 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) + +### Features + +* add 'extended' option to exporter ([#437](https://github.com/ipfs/js-ipfs-unixfs/issues/437)) ([332a794](https://github.com/ipfs/js-ipfs-unixfs/commit/332a794227f7792e1ddee1b1e47d01fd510d6cf4)) + ## [ipfs-unixfs-exporter-v13.6.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.5...ipfs-unixfs-exporter-13.6.6) (2025-06-18) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 7ec27009..29ddc64f 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.6.6", + "version": "13.7.0", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From c9a9bf45a5c8a779ed73cc2238a58c01e090edb7 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 31 Jul 2025 11:44:58 +0200 Subject: [PATCH 37/42] fix: add option to export non-extended unixfs (#438) Adds overrides for non-extended exports --- packages/ipfs-unixfs-exporter/src/index.ts | 52 ++++++++-------- .../resolvers/unixfs-v1/content/directory.ts | 15 ++--- .../content/hamt-sharded-directory.ts | 25 ++++---- .../src/resolvers/unixfs-v1/index.ts | 15 ++++- .../test/exporter-sharded.spec.ts | 3 +- .../test/exporter.spec.ts | 59 ++++++++++++++++++- 6 files changed, 114 insertions(+), 55 deletions(-) diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index ee6c2e9e..71019551 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -57,7 +57,6 @@ import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' -import type { AbortOptions } from 'it-pushable' import type { ProgressOptions, ProgressEvent } from 'progress-events' export * from './errors.js' @@ -314,6 +313,23 @@ export interface IdentityNode extends Exportable { */ export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode +export interface UnixFSBasicEntry { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID +} + export interface NextResult { cid: CID name: string @@ -327,39 +343,15 @@ export interface ResolveResult { } export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1BasicContent { - /** - * The name of the entry - */ - name: string - - /** - * The path of the entry within the DAG in which it was encountered - */ - path: string - - /** - * The CID of the entry - */ - cid: CID - - /** - * Resolve the root node of the entry to parse the UnixFS metadata contained - * there. The metadata will contain what kind of node it is (e.g. file, - * directory, etc), the file size, and more. - */ - resolve(options?: AbortOptions): Promise -} - export interface UnixFsV1ContentResolver { (options: ExporterOptions): UnixfsV1Content - (options: BasicExporterOptions): UnixfsV1BasicContent + (options: BasicExporterOptions): UnixFSBasicEntry } export interface UnixfsV1Resolver { @@ -435,6 +427,8 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * // entries contains 4x `entry` objects * ``` */ +export function walkPath (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function walkPath (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, @@ -491,6 +485,8 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag * } * ``` */ +export async function exporter (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): Promise export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) @@ -519,6 +515,8 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, * // entries contains all children of the `Qmfoo/foo/bar` directory and it's children * ``` */ +export function recursive (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function recursive (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index 614b33ca..73dc5c06 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -4,7 +4,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSBasicEntry, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { @@ -23,23 +23,18 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) - return result.entry - } - if (isBasicExporterOptions(options)) { - const basic: UnixfsV1BasicContent = { + const basic: UnixFSBasicEntry = { cid: link.Hash, name: linkName, - path: linkPath, - resolve: load + path: linkPath } return basic } - return load(options) + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) + return result.entry } }), source => parallel(source, { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index a3f56189..d191a688 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -6,7 +6,7 @@ import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSBasicEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { @@ -49,25 +49,26 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de if (name != null && name !== '') { const linkPath = `${path}/${name}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) - return result.entry - } if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name, + path: linkPath + } + return { - entries: [{ - cid: link.Hash, - name, - path: linkPath, - resolve: load - }] + entries: [ + basic + ] } } + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + return { entries: [ - await load() + result.entry ].filter(Boolean) } } else { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 4c5983e4..c847ca4c 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -2,10 +2,11 @@ import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' +import { isBasicExporterOptions } from '../../utils/is-basic-exporter-options.ts' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' -import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { Resolver, UnixFSBasicEntry, UnixfsV1Resolver } from '../../index.js' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' @@ -30,6 +31,18 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid, + name, + path + } + + return { + entry: basic + } + } + const block = await blockstore.get(cid, options) const node = decode(block) let unixfs diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index fe16201e..29fe8fc0 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -409,10 +409,9 @@ describe('exporter sharded', function () { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() } }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 9ce3056e..be9e58e7 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1606,7 +1606,7 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) - it('exports basic directory', async () => { + it('exports basic directory contents', async () => { const files: Record = {} for (let i = 0; i < 10; i++) { @@ -1649,10 +1649,63 @@ describe('exporter', () => { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file', async () => { + const imported = await all(importer([{ + content: uint8ArrayFromString('hello') + }], block, { + rawLeaves: false + })) + + const regularFile = await exporter(imported[0].cid, block) + expect(regularFile).to.have.property('unixfs') + + const basicFile = await exporter(imported[0].cid, block, { + extended: false + }) + + expect(basicFile).to.have.property('name') + expect(basicFile).to.have.property('path') + expect(basicFile).to.have.property('cid') + expect(basicFile).to.not.have.property('unixfs') + }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') } + + const basicDir = await exporter(dirCid, block, { + extended: false + }) + + expect(basicDir).to.have.property('name') + expect(basicDir).to.have.property('path') + expect(basicDir).to.have.property('cid') + expect(basicDir).to.not.have.property('unixfs') + expect(basicDir).to.not.have.property('content') }) }) From 9509fb7f341240417f839c0919bba90c4fe54093 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 31 Jul 2025 09:50:27 +0000 Subject: [PATCH 38/42] chore(release): 13.7.1 [skip ci] ## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) ### Bug Fixes * add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index bc190919..e31ff5e4 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) + +### Bug Fixes + +* add option to export non-extended unixfs ([#438](https://github.com/ipfs/js-ipfs-unixfs/issues/438)) ([c9a9bf4](https://github.com/ipfs/js-ipfs-unixfs/commit/c9a9bf45a5c8a779ed73cc2238a58c01e090edb7)) + ## [ipfs-unixfs-exporter-v13.7.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.6.6...ipfs-unixfs-exporter-13.7.0) (2025-07-30) ### Features diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 29ddc64f..5df17ca1 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.0", + "version": "13.7.1", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From 278aea4c2ed76a8d890a0d2d3a079b03a9c00334 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 31 Jul 2025 12:15:25 +0200 Subject: [PATCH 39/42] fix: add extended to default exporter options (#439) Otherwise we can't pass true or undefined. --- packages/ipfs-unixfs-exporter/src/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index 71019551..35e92277 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -134,6 +134,19 @@ export interface ExporterOptions extends ProgressOptions * (default: undefined) */ blockReadConcurrency?: number + + /** + * When directory contents are listed, by default the root node of each entry + * is fetched to decode the UnixFS metadata and know if the entry is a file or + * a directory. This can result in fetching extra data which may not be + * desirable, depending on your application. + * + * Pass false here to only return the CID and the name of the entry and not + * any extended metadata. + * + * @default true + */ + extended?: boolean } export interface BasicExporterOptions extends ExporterOptions { From e3fbc9672a6b6223fcac839fdf691f4c8bcbdca5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 31 Jul 2025 10:21:14 +0000 Subject: [PATCH 40/42] chore(release): 13.7.2 [skip ci] ## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) ### Bug Fixes * add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index e31ff5e4..7fbdf258 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) + +### Bug Fixes + +* add extended to default exporter options ([#439](https://github.com/ipfs/js-ipfs-unixfs/issues/439)) ([278aea4](https://github.com/ipfs/js-ipfs-unixfs/commit/278aea4c2ed76a8d890a0d2d3a079b03a9c00334)) + ## [ipfs-unixfs-exporter-v13.7.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.0...ipfs-unixfs-exporter-13.7.1) (2025-07-31) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 5df17ca1..5d4cae58 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.1", + "version": "13.7.2", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme", From b8d33deb0dfc76cc53eb82e31a67748a8da24eae Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 12 Aug 2025 07:56:24 +0100 Subject: [PATCH 41/42] fix: export basic file from dir or shard (#440) Fixes a bug whereby when exporting a deep path to a file within a directory or shard, the returned entry had the CID of the containing folder rather than the target file. --- .../src/resolvers/unixfs-v1/index.ts | 2 +- .../test/exporter-sharded.spec.ts | 33 +++++++++++++++++++ .../test/exporter.spec.ts | 29 ++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index c847ca4c..4fd337f3 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -31,7 +31,7 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { - if (isBasicExporterOptions(options)) { + if (isBasicExporterOptions(options) && toResolve.length === 0) { const basic: UnixFSBasicEntry = { cid, name, diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index 29fe8fc0..a6abbc85 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -414,4 +414,37 @@ describe('exporter sharded', function () { await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() } }) + + it('exports basic file from sharded directory', async () => { + const files: Record = {} + + // needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes + for (let i = 0; i < 100; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index be9e58e7..7d4de993 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1708,4 +1708,33 @@ describe('exporter', () => { expect(basicDir).to.not.have.property('unixfs') expect(basicDir).to.not.have.property('content') }) + + it('exports basic file from directory', async () => { + const files: Record = { + 'file.txt': { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const file = imported[0] + const dir = imported[imported.length - 1] + + const basicfile = await exporter(`/ipfs/${dir.cid}/${file.path}`, block, { + extended: false + }) + + expect(basicfile).to.have.property('name', file.path) + expect(basicfile).to.have.property('path', `${dir.cid}/${file.path}`) + expect(basicfile).to.have.deep.property('cid', file.cid) + expect(basicfile).to.not.have.property('unixfs') + expect(basicfile).to.not.have.property('content') + }) }) From 7f15bafe6d9efc58a42eeb26ec5165c332493927 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 12 Aug 2025 07:01:59 +0000 Subject: [PATCH 42/42] chore(release): 13.7.3 [skip ci] ## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) ### Bug Fixes * export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) --- packages/ipfs-unixfs-exporter/CHANGELOG.md | 6 ++++++ packages/ipfs-unixfs-exporter/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 7fbdf258..463832da 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,9 @@ +## [ipfs-unixfs-exporter-v13.7.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.2...ipfs-unixfs-exporter-13.7.3) (2025-08-12) + +### Bug Fixes + +* export basic file from dir or shard ([#440](https://github.com/ipfs/js-ipfs-unixfs/issues/440)) ([b8d33de](https://github.com/ipfs/js-ipfs-unixfs/commit/b8d33deb0dfc76cc53eb82e31a67748a8da24eae)) + ## [ipfs-unixfs-exporter-v13.7.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-13.7.1...ipfs-unixfs-exporter-13.7.2) (2025-07-31) ### Bug Fixes diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 5d4cae58..1f3a4177 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "13.7.2", + "version": "13.7.3", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/main/packages/ipfs-unixfs-exporter#readme",